feat: impl StructArray -- support dynamic add struct field (#49807)

issue: https://github.com/milvus-io/milvus/issues/42148
design doc: docs/design-docs/design_docs/20260306-struct.md

this PR also fixes https://github.com/milvus-io/milvus/issues/49693

---------

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
This commit is contained in:
Spade A
2026-05-17 01:02:29 +08:00
committed by GitHub
parent 9f501f9bea
commit b944be4a31
45 changed files with 2191 additions and 692 deletions
+3 -3
View File
@@ -7,15 +7,13 @@ require (
github.com/cockroachdb/errors v1.9.1
github.com/google/uuid v1.6.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
github.com/quasilyte/go-ruleguard/dsl v0.3.23
github.com/samber/lo v1.52.0
github.com/stretchr/testify v1.11.1
github.com/tidwall/gjson v1.17.1
go.opentelemetry.io/otel v1.43.0
go.uber.org/atomic v1.11.0
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
google.golang.org/grpc v1.80.0
google.golang.org/protobuf v1.36.11
)
@@ -95,6 +93,7 @@ require (
go.etcd.io/etcd/server/v3 v3.5.23 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect
go.opentelemetry.io/otel v1.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
go.opentelemetry.io/otel/metric v1.43.0 // indirect
@@ -105,6 +104,7 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.49.0 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/net v0.52.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.42.0 // indirect
+2 -4
View File
@@ -222,10 +222,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -86,6 +86,65 @@ func (_c *MilvusServiceServer_AddCollectionField_Call) RunAndReturn(run func(con
return _c
}
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
func (_m *MilvusServiceServer) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for AddCollectionStructField")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MilvusServiceServer_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
type MilvusServiceServer_AddCollectionStructField_Call struct {
*mock.Call
}
// AddCollectionStructField is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.AddCollectionStructFieldRequest
func (_e *MilvusServiceServer_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MilvusServiceServer_AddCollectionStructField_Call {
return &MilvusServiceServer_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
}
func (_c *MilvusServiceServer_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MilvusServiceServer_AddCollectionStructField_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
})
return _c
}
func (_c *MilvusServiceServer_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MilvusServiceServer_AddCollectionStructField_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MilvusServiceServer_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MilvusServiceServer_AddCollectionStructField_Call {
_c.Call.Return(run)
return _c
}
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
func (_m *MilvusServiceServer) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
+1 -1
View File
@@ -69,7 +69,7 @@ require (
github.com/hamba/avro/v2 v2.29.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/magiconair/properties v1.8.7
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
github.com/milvus-io/milvus/client/v2 v2.6.2
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
github.com/shirou/gopsutil/v4 v4.25.10
+2 -4
View File
@@ -814,10 +814,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
+4
View File
@@ -575,6 +575,10 @@ func (c *mixCoordImpl) AddCollectionField(ctx context.Context, in *milvuspb.AddC
return c.rootcoordServer.AddCollectionField(ctx, in)
}
func (c *mixCoordImpl) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
return c.rootcoordServer.AddCollectionStructField(ctx, in)
}
func (s *mixCoordImpl) CreateCredential(ctx context.Context, req *internalpb.CredentialInfo) (*commonpb.Status, error) {
return s.rootcoordServer.CreateCredential(ctx, req)
}
+9
View File
@@ -50,6 +50,7 @@
#include "index/VectorDiskIndex.h"
#include "index/VectorMemIndex.h"
#include "knowhere/comp/knowhere_check.h"
#include "knowhere/emb_list_utils.h"
#include "knowhere/expected.h"
#include "knowhere/index/index_static.h"
#include "knowhere/operands.h"
@@ -256,6 +257,9 @@ IndexFactory::VecIndexLoadResource(
index_type, index_version, config);
break;
case milvus::DataType::VECTOR_ARRAY: {
auto metric_type = milvus::index::GetMetricTypeFromConfig(config);
auto is_emb_list_metric =
knowhere::get_el_metric_type(metric_type).has_value();
switch (element_type) {
case milvus::DataType::VECTOR_FLOAT:
resource = knowhere::IndexStaticFaced<knowhere::fp32>::
@@ -326,6 +330,11 @@ IndexFactory::VecIndexLoadResource(
element_type);
return LoadResourceRequest{0, 0, 0, 0, true};
}
// Non-emb-list VECTOR_ARRAY indexes do not keep embedding-list
// offsets, so they cannot reconstruct row-level embedding lists.
if (!is_emb_list_metric) {
has_raw_data = false;
}
break;
}
default:
@@ -3418,12 +3418,31 @@ ChunkedSegmentSealedImpl::get_raw_data(milvus::OpContext* op_ctx,
break;
}
case DataType::VECTOR_ARRAY: {
bulk_subscript_vector_array_impl(
op_ctx,
column.get(),
valid_offsets,
valid_count,
ret->mutable_vectors()->mutable_vector_array()->mutable_data());
auto dst =
ret->mutable_vectors()->mutable_vector_array()->mutable_data();
if (field_meta.is_nullable() && valid_data != nullptr) {
if (valid_count == 0) {
break;
}
std::vector<int64_t> valid_logical_offsets;
valid_logical_offsets.reserve(valid_count);
for (int64_t i = 0; i < count; ++i) {
if (valid_data[i]) {
valid_logical_offsets.push_back(i);
}
}
column->BulkVectorArrayAt(
op_ctx,
[dst, &valid_logical_offsets](VectorFieldProto&& array,
size_t i) {
dst->at(valid_logical_offsets[i]) = std::move(array);
},
valid_offsets,
valid_count);
} else {
bulk_subscript_vector_array_impl(
op_ctx, column.get(), seg_offsets, count, dst);
}
break;
}
default: {
@@ -774,7 +774,8 @@ SegmentInternalInterface::bulk_subscript_not_exist_field(
AssertInfo(field_meta.is_nullable(),
"Non-nullable vector field should not reach here");
auto result = CreateEmptyVectorDataArray(0, field_meta);
auto create_count = IsVectorArrayDataType(data_type) ? count : 0;
auto result = CreateEmptyVectorDataArray(create_count, field_meta);
auto valid_data = result->mutable_valid_data();
for (int64_t i = 0; i < count; ++i) {
+3 -1
View File
@@ -501,6 +501,7 @@ CreateEmptyVectorDataArray(int64_t count, const FieldMeta& field_meta) {
}
case DataType::VECTOR_ARRAY: {
auto obj = vector_array->mutable_vector_array();
obj->set_dim(dim);
obj->set_element_type(static_cast<milvus::proto::schema::DataType>(
field_meta.get_element_type()));
obj->mutable_data()->Reserve(count);
@@ -523,7 +524,8 @@ CreateEmptyVectorDataArray(int64_t count,
int64_t valid_count,
const void* valid_data,
const FieldMeta& field_meta) {
int64_t data_count = (field_meta.is_nullable() && valid_data != nullptr)
int64_t data_count = (field_meta.is_nullable() && valid_data != nullptr &&
field_meta.get_data_type() != DataType::VECTOR_ARRAY)
? valid_count
: count;
auto data_array = CreateEmptyVectorDataArray(data_count, field_meta);
@@ -184,6 +184,12 @@ DefaultValueChunkTranslator::value_size() const {
case milvus::DataType::ARRAY:
value_size = sizeof(Array);
break;
case milvus::DataType::VECTOR_ARRAY:
AssertInfo(field_meta_.is_nullable(),
"only nullable vector array fields can be "
"dynamically added");
value_size = 0;
break;
case milvus::DataType::VECTOR_FLOAT:
case milvus::DataType::VECTOR_BINARY:
case milvus::DataType::VECTOR_FLOAT16:
@@ -231,7 +237,16 @@ DefaultValueChunkTranslator::build_buffer_for_rows(
auto data_type = field_meta_.get_data_type();
std::shared_ptr<arrow::ArrayBuilder> builder;
if (IsVectorDataType(data_type)) {
if (data_type == milvus::DataType::VECTOR_ARRAY) {
AssertInfo(field_meta_.is_nullable(),
"only nullable vector array fields can be "
"dynamically added");
builder =
milvus::storage::CreateArrowBuilder(data_type,
field_meta_.get_element_type(),
field_meta_.get_dim(),
true);
} else if (IsVectorDataType(data_type)) {
AssertInfo(field_meta_.is_nullable(),
"only nullable vector fields can be dynamically added");
builder = std::make_shared<arrow::BinaryBuilder>();
@@ -1091,6 +1091,49 @@ TEST_P(DefaultValueChunkTranslatorTest, TestNullableVectorFloat) {
EXPECT_EQ(total_rows, row_count);
}
// Test nullable VECTOR_ARRAY with all-null default values.
TEST_P(DefaultValueChunkTranslatorTest, TestNullableVectorArray) {
int64_t dim = 4;
int64_t row_count = 100;
FieldMeta field_meta(FieldName("test_vector_array_nullable"),
FieldId(1302),
DataType::VECTOR_ARRAY,
DataType::VECTOR_FLOAT,
dim,
std::nullopt, // metric_type
true); // nullable
FieldDataInfo field_data_info(1302, row_count, getMmapDirPath());
auto translator = std::make_unique<DefaultValueChunkTranslator>(
segment_id_, field_meta, field_data_info, GetParam(), true);
EXPECT_EQ(translator->value_size(), 0);
size_t num_cells = translator->num_cells();
ASSERT_GE(num_cells, 1);
std::vector<cachinglayer::cid_t> cids;
for (size_t i = 0; i < num_cells; ++i) {
cids.push_back(i);
}
auto cells = translator->get_cells(nullptr, cids);
EXPECT_EQ(cells.size(), num_cells);
int64_t total_rows = 0;
for (auto& [cid, chunk] : cells) {
total_rows += chunk->RowNums();
for (int64_t i = 0; i < chunk->RowNums(); ++i) {
EXPECT_FALSE(chunk->isValid(i))
<< "Expected null at cell " << cid << " row " << i;
}
}
EXPECT_EQ(total_rows, row_count);
}
// Test nullable VECTOR_SPARSE_U32_F32 with all-null default values
TEST_P(DefaultValueChunkTranslatorTest, TestNullableSparseVector) {
bool use_mmap = GetParam();
+4
View File
@@ -220,6 +220,10 @@ func (m *mockMixCoord) AddCollectionField(ctx context.Context, req *milvuspb.Add
panic("implement me")
}
func (m *mockMixCoord) AddCollectionStructField(ctx context.Context, req *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
panic("implement me")
}
func (m *mockMixCoord) GetQuotaMetrics(ctx context.Context, req *internalpb.GetQuotaMetricsRequest) (*internalpb.GetQuotaMetricsResponse, error) {
panic("implement me")
}
@@ -259,6 +259,17 @@ func (c *Client) AddCollectionField(ctx context.Context, in *milvuspb.AddCollect
})
}
func (c *Client) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
in = typeutil.Clone(in)
commonpbutil.UpdateMsgBase(
in.GetBase(),
commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())),
)
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*commonpb.Status, error) {
return client.AddCollectionStructField(ctx, in)
})
}
// DescribeCollection return collection info
func (c *Client) DescribeCollection(ctx context.Context, in *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error) {
in = typeutil.Clone(in)
+4
View File
@@ -410,6 +410,10 @@ func (s *Server) AddCollectionField(ctx context.Context, in *milvuspb.AddCollect
return s.mixCoord.AddCollectionField(ctx, in)
}
func (s *Server) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
return s.mixCoord.AddCollectionStructField(ctx, in)
}
// CreatePartition creates a partition in a collection
func (s *Server) CreatePartition(ctx context.Context, in *milvuspb.CreatePartitionRequest) (*commonpb.Status, error) {
return s.mixCoord.CreatePartition(ctx, in)
+5
View File
@@ -668,6 +668,11 @@ func (s *Server) AddCollectionField(ctx context.Context, request *milvuspb.AddCo
return s.proxy.AddCollectionField(ctx, request)
}
// AddCollectionStructField add a struct field to collection
func (s *Server) AddCollectionStructField(ctx context.Context, request *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
return s.proxy.AddCollectionStructField(ctx, request)
}
// GetCollectionStatistics notifies Proxy to get a collection's Statistics
func (s *Server) GetCollectionStatistics(ctx context.Context, request *milvuspb.GetCollectionStatisticsRequest) (*milvuspb.GetCollectionStatisticsResponse, error) {
return s.proxy.GetCollectionStatistics(ctx, request)
+59
View File
@@ -164,6 +164,65 @@ func (_c *MixCoord_AddCollectionField_Call) RunAndReturn(run func(context.Contex
return _c
}
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
func (_m *MixCoord) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for AddCollectionStructField")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MixCoord_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
type MixCoord_AddCollectionStructField_Call struct {
*mock.Call
}
// AddCollectionStructField is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.AddCollectionStructFieldRequest
func (_e *MixCoord_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MixCoord_AddCollectionStructField_Call {
return &MixCoord_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
}
func (_c *MixCoord_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MixCoord_AddCollectionStructField_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
})
return _c
}
func (_c *MixCoord_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MixCoord_AddCollectionStructField_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MixCoord_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MixCoord_AddCollectionStructField_Call {
_c.Call.Return(run)
return _c
}
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
func (_m *MixCoord) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
+74
View File
@@ -187,6 +187,80 @@ func (_c *MockMixCoordClient_AddCollectionField_Call) RunAndReturn(run func(cont
return _c
}
// AddCollectionStructField provides a mock function with given fields: ctx, in, opts
func (_m *MockMixCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, in)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for AddCollectionStructField")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
return rf(ctx, in, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) *commonpb.Status); ok {
r0 = rf(ctx, in, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) error); ok {
r1 = rf(ctx, in, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMixCoordClient_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
type MockMixCoordClient_AddCollectionStructField_Call struct {
*mock.Call
}
// AddCollectionStructField is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.AddCollectionStructFieldRequest
// - opts ...grpc.CallOption
func (_e *MockMixCoordClient_Expecter) AddCollectionStructField(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_AddCollectionStructField_Call {
return &MockMixCoordClient_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField",
append([]interface{}{ctx, in}, opts...)...)}
}
func (_c *MockMixCoordClient_AddCollectionStructField_Call) Run(run func(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption)) *MockMixCoordClient_AddCollectionStructField_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]grpc.CallOption, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(grpc.CallOption)
}
}
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest), variadicArgs...)
})
return _c
}
func (_c *MockMixCoordClient_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockMixCoordClient_AddCollectionStructField_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMixCoordClient_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockMixCoordClient_AddCollectionStructField_Call {
_c.Call.Return(run)
return _c
}
// AddCollectionFunction provides a mock function with given fields: ctx, in, opts
func (_m *MockMixCoordClient) AddCollectionFunction(ctx context.Context, in *milvuspb.AddCollectionFunctionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts))
+59
View File
@@ -92,6 +92,65 @@ func (_c *MockProxy_AddCollectionField_Call) RunAndReturn(run func(context.Conte
return _c
}
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
func (_m *MockProxy) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for AddCollectionStructField")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockProxy_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
type MockProxy_AddCollectionStructField_Call struct {
*mock.Call
}
// AddCollectionStructField is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.AddCollectionStructFieldRequest
func (_e *MockProxy_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MockProxy_AddCollectionStructField_Call {
return &MockProxy_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
}
func (_c *MockProxy_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MockProxy_AddCollectionStructField_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
})
return _c
}
func (_c *MockProxy_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockProxy_AddCollectionStructField_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockProxy_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MockProxy_AddCollectionStructField_Call {
_c.Call.Return(run)
return _c
}
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
func (_m *MockProxy) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
+59
View File
@@ -95,6 +95,65 @@ func (_c *MockRootCoord_AddCollectionField_Call) RunAndReturn(run func(context.C
return _c
}
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
func (_m *MockRootCoord) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for AddCollectionStructField")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRootCoord_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
type MockRootCoord_AddCollectionStructField_Call struct {
*mock.Call
}
// AddCollectionStructField is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.AddCollectionStructFieldRequest
func (_e *MockRootCoord_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MockRootCoord_AddCollectionStructField_Call {
return &MockRootCoord_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
}
func (_c *MockRootCoord_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MockRootCoord_AddCollectionStructField_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
})
return _c
}
func (_c *MockRootCoord_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockRootCoord_AddCollectionStructField_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockRootCoord_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MockRootCoord_AddCollectionStructField_Call {
_c.Call.Return(run)
return _c
}
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
func (_m *MockRootCoord) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
+74
View File
@@ -107,6 +107,80 @@ func (_c *MockRootCoordClient_AddCollectionField_Call) RunAndReturn(run func(con
return _c
}
// AddCollectionStructField provides a mock function with given fields: ctx, in, opts
func (_m *MockRootCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, in)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for AddCollectionStructField")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
return rf(ctx, in, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) *commonpb.Status); ok {
r0 = rf(ctx, in, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) error); ok {
r1 = rf(ctx, in, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockRootCoordClient_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
type MockRootCoordClient_AddCollectionStructField_Call struct {
*mock.Call
}
// AddCollectionStructField is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.AddCollectionStructFieldRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) AddCollectionStructField(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AddCollectionStructField_Call {
return &MockRootCoordClient_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField",
append([]interface{}{ctx, in}, opts...)...)}
}
func (_c *MockRootCoordClient_AddCollectionStructField_Call) Run(run func(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption)) *MockRootCoordClient_AddCollectionStructField_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]grpc.CallOption, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(grpc.CallOption)
}
}
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest), variadicArgs...)
})
return _c
}
func (_c *MockRootCoordClient_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockRootCoordClient_AddCollectionStructField_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockRootCoordClient_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockRootCoordClient_AddCollectionStructField_Call {
_c.Call.Return(run)
return _c
}
// AddCollectionFunction provides a mock function with given fields: ctx, in, opts
func (_m *MockRootCoordClient) AddCollectionFunction(ctx context.Context, in *milvuspb.AddCollectionFunctionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts))
+5
View File
@@ -306,6 +306,11 @@ func fillDatabase(ctx context.Context, req interface{}) (context.Context, interf
r.DbName = GetCurDBNameFromContextOrDefault(ctx)
}
return ctx, r
case *milvuspb.AddCollectionStructFieldRequest:
if r.DbName == "" {
r.DbName = GetCurDBNameFromContextOrDefault(ctx)
}
return ctx, r
case *milvuspb.AlterCollectionSchemaRequest:
if r.DbName == "" {
r.DbName = GetCurDBNameFromContextOrDefault(ctx)
@@ -102,6 +102,7 @@ func TestDatabaseInterceptor(t *testing.T) {
&milvuspb.SelectGrantRequest{Entity: &milvuspb.GrantEntity{}},
&milvuspb.ManualCompactionRequest{},
&milvuspb.AddCollectionFieldRequest{},
&milvuspb.AddCollectionStructFieldRequest{},
&milvuspb.AlterCollectionSchemaRequest{},
&milvuspb.RunAnalyzerRequest{},
&milvuspb.RefreshExternalCollectionRequest{},
+69
View File
@@ -1027,6 +1027,75 @@ func (node *Proxy) AddCollectionField(ctx context.Context, request *milvuspb.Add
return task.result, nil
}
// AddCollectionStructField add a struct field to collection
func (node *Proxy) AddCollectionStructField(ctx context.Context, request *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
return merr.Status(err), nil
}
ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, "Proxy-AddCollectionStructField")
defer sp.End()
dresp, err := node.DescribeCollection(ctx, &milvuspb.DescribeCollectionRequest{DbName: request.DbName, CollectionName: request.CollectionName})
if err := merr.CheckRPCCall(dresp, err); err != nil {
return merr.Status(err), nil
}
if typeutil.IsExternalCollection(dresp.GetSchema()) {
return merr.Status(merr.WrapErrParameterInvalidMsg(
"add struct field operation is not supported for external collection %s", request.GetCollectionName())), nil
}
task := &addCollectionStructFieldTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
AddCollectionStructFieldRequest: request,
mixCoord: node.mixCoord,
oldSchema: dresp.GetSchema(),
}
method := "AddCollectionStructField"
tr := timerecord.NewTimeRecorder(method)
log := log.Ctx(ctx).With(
zap.String("role", typeutil.ProxyRole),
zap.String("db", request.DbName),
zap.String("collection", request.CollectionName))
log.Info(rpcReceived(method))
if err := node.sched.ddQueue.Enqueue(task); err != nil {
log.Warn(
rpcFailedToEnqueue(method),
zap.Error(err))
return merr.Status(err), nil
}
log.Info(
rpcEnqueued(method),
zap.Uint64("BeginTs", task.BeginTs()),
zap.Uint64("EndTs", task.EndTs()))
if err := task.WaitToFinish(); err != nil {
log.Warn(
rpcFailedToWaitToFinish(method),
zap.Error(err),
zap.Uint64("BeginTs", task.BeginTs()),
zap.Uint64("EndTs", task.EndTs()))
return merr.Status(err), nil
}
log.Info(
rpcDone(method),
zap.Uint64("BeginTs", task.BeginTs()),
zap.Uint64("EndTs", task.EndTs()))
metrics.ProxyReqLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method).Observe(float64(tr.ElapseSpan().Milliseconds()))
return task.result, nil
}
func (node *Proxy) AlterCollectionSchema(ctx context.Context, request *milvuspb.AlterCollectionSchemaRequest) (*milvuspb.AlterCollectionSchemaResponse, error) {
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
return &milvuspb.AlterCollectionSchemaResponse{
+4
View File
@@ -261,6 +261,10 @@ func (c *MockMixCoordClientInterface) AddCollectionField(ctx context.Context, in
panic("implement me")
}
func (c *MockMixCoordClientInterface) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
panic("implement me")
}
// describeCollectionInternal return collection info
func (c *MockMixCoordClientInterface) describeCollectionInternal(ctx context.Context, in *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error) {
panic("implement me")
+48
View File
@@ -394,6 +394,54 @@ func (coord *MixCoordMock) AddCollectionField(ctx context.Context, req *milvuspb
return merr.Success(), nil
}
func (coord *MixCoordMock) AddCollectionStructField(ctx context.Context, req *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
code := coord.state.Load().(commonpb.StateCode)
if code != commonpb.StateCode_Healthy {
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: fmt.Sprintf("state code = %s", commonpb.StateCode_name[int32(code)]),
}, nil
}
coord.collMtx.Lock()
defer coord.collMtx.Unlock()
collID, exist := coord.collName2ID[req.CollectionName]
if !exist {
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_CollectionNotExists,
Reason: "collection not exist",
}, nil
}
collInfo, exist := coord.collID2Meta[collID]
if !exist {
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_CollectionNotExists,
Reason: "collection info not exist",
}, nil
}
structField := proto.Clone(req.GetStructArrayFieldSchema()).(*schemapb.StructArrayFieldSchema)
fieldIDStart := int64(common.StartOfUserFieldID + typeutil.GetTotalFieldsNum(collInfo.schema) + 1)
structField.FieldID = fieldIDStart
for i, field := range structField.GetFields() {
field.FieldID = fieldIDStart + int64(i) + 1
}
collInfo.schema.StructArrayFields = append(collInfo.schema.StructArrayFields, structField)
ts := uint64(time.Now().Nanosecond())
coord.collID2Meta[collID] = collectionMeta{
name: req.CollectionName,
id: collID,
schema: collInfo.schema,
shardsNum: collInfo.shardsNum,
virtualChannelNames: collInfo.virtualChannelNames,
physicalChannelNames: collInfo.physicalChannelNames,
createdTimestamp: ts,
createdUtcTimestamp: ts,
properties: collInfo.properties,
}
return merr.Success(), nil
}
func (coord *MixCoordMock) CreateCollection(ctx context.Context, req *milvuspb.CreateCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
code := coord.state.Load().(commonpb.StateCode)
if code != commonpb.StateCode_Healthy {
+210 -3
View File
@@ -133,6 +133,7 @@ const (
DescribeDatabaseTaskName = "DescribeDatabaseTaskName"
AddFieldTaskName = "AddFieldTaskName"
AddStructFieldTaskName = "AddStructFieldTaskName"
AlterCollectionSchemaTaskName = "AlterCollectionSchemaTaskName"
// minFloat32 minimum float.
@@ -700,15 +701,221 @@ func (t *addCollectionFieldTask) PostExecute(ctx context.Context) error {
return nil
}
type addCollectionStructFieldTask struct {
baseTask
Condition
*milvuspb.AddCollectionStructFieldRequest
ctx context.Context
mixCoord types.MixCoordClient
result *commonpb.Status
structFieldSchema *schemapb.StructArrayFieldSchema
oldSchema *schemapb.CollectionSchema
}
func (t *addCollectionStructFieldTask) TraceCtx() context.Context {
return t.ctx
}
func (t *addCollectionStructFieldTask) ID() UniqueID {
return t.Base.MsgID
}
func (t *addCollectionStructFieldTask) SetID(uid UniqueID) {
t.Base.MsgID = uid
}
func (t *addCollectionStructFieldTask) Name() string {
return AddStructFieldTaskName
}
func (t *addCollectionStructFieldTask) Type() commonpb.MsgType {
return t.Base.MsgType
}
func (t *addCollectionStructFieldTask) BeginTs() Timestamp {
return t.Base.Timestamp
}
func (t *addCollectionStructFieldTask) EndTs() Timestamp {
return t.Base.Timestamp
}
func (t *addCollectionStructFieldTask) SetTs(ts Timestamp) {
t.Base.Timestamp = ts
}
func (t *addCollectionStructFieldTask) OnEnqueue() error {
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
t.Base.MsgType = commonpb.MsgType_AddCollectionField
t.Base.SourceID = paramtable.GetNodeID()
return nil
}
func (t *addCollectionStructFieldTask) PreExecute(ctx context.Context) error {
if t.oldSchema == nil {
return merr.WrapErrParameterInvalidMsg("empty old schema in add struct field task")
}
if t.GetStructArrayFieldSchema() == nil {
return merr.WrapErrParameterInvalidMsg("struct array field schema is nil")
}
t.structFieldSchema = proto.Clone(t.GetStructArrayFieldSchema()).(*schemapb.StructArrayFieldSchema)
if err := validateAddStructFieldRequest(t.oldSchema, t.structFieldSchema); err != nil {
return err
}
transformStructArrayFieldSubNames(t.structFieldSchema)
t.StructArrayFieldSchema = t.structFieldSchema
log.Info("PreExecute addStructField task done", zap.Any("struct field schema", t.structFieldSchema))
return nil
}
func (t *addCollectionStructFieldTask) Execute(ctx context.Context) error {
var err error
t.result, err = t.mixCoord.AddCollectionStructField(ctx, t.AddCollectionStructFieldRequest)
return merr.CheckRPCCall(t.result, err)
}
func (t *addCollectionStructFieldTask) PostExecute(ctx context.Context) error {
return nil
}
func validateAddStructFieldRequest(schema *schemapb.CollectionSchema, structFieldSchema *schemapb.StructArrayFieldSchema) error {
if schema == nil {
return merr.WrapErrParameterInvalidMsg("empty old schema in add struct field task")
}
if structFieldSchema == nil {
return merr.WrapErrParameterInvalidMsg("struct array field schema is nil")
}
if !structFieldSchema.GetNullable() {
return merr.WrapErrParameterInvalidMsg("added struct field must be nullable, please check it, struct field name = %s", structFieldSchema.GetName())
}
if err := validateFieldName(structFieldSchema.GetName()); err != nil {
return err
}
if funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, structFieldSchema.GetName()) {
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", structFieldSchema.GetName())
}
if err := ValidateStructArrayField(structFieldSchema, schema); err != nil {
return err
}
if err := validateAddStructSubFieldProperties(structFieldSchema); err != nil {
return err
}
if err := validateAddStructFieldNames(schema, structFieldSchema); err != nil {
return err
}
totalFieldsNum := typeutil.GetTotalFieldsNum(schema) + len(structFieldSchema.GetFields()) + 1
if totalFieldsNum > Params.ProxyCfg.MaxFieldNum.GetAsInt() {
return fmt.Errorf("maximum field's number should be limited to %d", Params.ProxyCfg.MaxFieldNum.GetAsInt())
}
vectorFields := len(typeutil.GetVectorFieldSchemas(schema))
for _, subField := range structFieldSchema.GetFields() {
if typeutil.IsVectorType(subField.GetDataType()) {
vectorFields++
}
}
if vectorFields > Params.ProxyCfg.MaxVectorFieldNum.GetAsInt() {
return fmt.Errorf("maximum vector field's number should be limited to %d", Params.ProxyCfg.MaxVectorFieldNum.GetAsInt())
}
return nil
}
func validateAddStructSubFieldProperties(structFieldSchema *schemapb.StructArrayFieldSchema) error {
for _, subField := range structFieldSchema.GetFields() {
if funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, subField.GetName()) {
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", subField.GetName())
}
if subField.GetIsPrimaryKey() {
return merr.WrapErrParameterInvalidMsg("primary key is not supported for struct field, field name = %s", subField.GetName())
}
if subField.GetAutoID() {
return merr.WrapErrParameterInvalidMsg("autoID is not supported for struct field, field name = %s", subField.GetName())
}
if subField.GetIsPartitionKey() {
return merr.WrapErrParameterInvalidMsg("partition key is not supported for struct field, field name = %s", subField.GetName())
}
if subField.GetIsClusteringKey() {
return merr.WrapErrParameterInvalidMsg("clustering key is not supported for struct field, field name = %s", subField.GetName())
}
if subField.GetDefaultValue() != nil {
return merr.WrapErrParameterInvalidMsg("default value is not supported for struct field, field name = %s", subField.GetName())
}
if subField.GetIsFunctionOutput() {
return merr.WrapErrParameterInvalidMsg("function output is not supported for struct field, field name = %s", subField.GetName())
}
if subField.GetExternalField() != "" {
return merr.WrapErrParameterInvalidMsg("add struct field operation does not support external field mapping, field name = %s", subField.GetName())
}
}
return nil
}
func validateAddStructFieldNames(schema *schemapb.CollectionSchema, structFieldSchema *schemapb.StructArrayFieldSchema) error {
existingNames := typeutil.NewSet[string]()
for _, field := range schema.GetFields() {
existingNames.Insert(field.GetName())
}
for _, structArrayField := range schema.GetStructArrayFields() {
existingNames.Insert(structArrayField.GetName())
for _, subField := range structArrayField.GetFields() {
if typeutil.IsStructSubField(subField.GetName()) {
existingNames.Insert(subField.GetName())
}
existingNames.Insert(storedStructSubFieldName(structArrayField.GetName(), subField.GetName()))
}
}
if existingNames.Contain(structFieldSchema.GetName()) {
return merr.WrapErrParameterInvalidMsg("duplicated field name %s", structFieldSchema.GetName())
}
newSubFieldNames := make(map[string]struct{})
for _, subField := range structFieldSchema.GetFields() {
if _, ok := newSubFieldNames[subField.GetName()]; ok {
return merr.WrapErrParameterInvalidMsg("duplicated field name %s", subField.GetName())
}
newSubFieldNames[subField.GetName()] = struct{}{}
transformedName := typeutil.ConcatStructFieldName(structFieldSchema.GetName(), subField.GetName())
if existingNames.Contain(transformedName) {
return merr.WrapErrParameterInvalidMsg("duplicated field name %s", transformedName)
}
}
return nil
}
func storedStructSubFieldName(structName string, fieldName string) string {
if typeutil.IsStructSubField(fieldName) {
return fieldName
}
return typeutil.ConcatStructFieldName(structName, fieldName)
}
func transformStructArrayFieldSubNames(structFieldSchema *schemapb.StructArrayFieldSchema) {
structName := structFieldSchema.GetName()
for _, field := range structFieldSchema.GetFields() {
field.Name = typeutil.ConcatStructFieldName(structName, field.GetName())
}
}
// validateAddFieldRequest validates both the old schema constraints and the new field properties
// for an AddCollectionField request. It is the single source of truth for add-field validation.
func validateAddFieldRequest(schema *schemapb.CollectionSchema, newFieldSchema *schemapb.FieldSchema) error {
// --- old schema constraints ---
fieldList := typeutil.NewSet[string]()
for _, f := range schema.Fields {
fieldList.Insert(f.Name)
for _, field := range schema.GetFields() {
fieldList.Insert(field.GetName())
}
if len(fieldList) >= Params.ProxyCfg.MaxFieldNum.GetAsInt() {
for _, structArrayField := range schema.GetStructArrayFields() {
fieldList.Insert(structArrayField.GetName())
}
if typeutil.GetTotalFieldsNum(schema) >= Params.ProxyCfg.MaxFieldNum.GetAsInt() {
msg := fmt.Sprintf("The number of fields has reached the maximum value %d", Params.ProxyCfg.MaxFieldNum.GetAsInt())
return merr.WrapErrParameterInvalidMsg(msg)
}
+136
View File
@@ -1257,6 +1257,142 @@ func TestAddFieldTask(t *testing.T) {
})
}
func TestAddCollectionStructFieldTaskPreExecute(t *testing.T) {
ctx := context.Background()
oldSchema := &schemapb.CollectionSchema{
Name: "test_collection",
Fields: []*schemapb.FieldSchema{
{
FieldID: 100,
Name: "pk",
DataType: schemapb.DataType_Int64,
IsPrimaryKey: true,
},
},
}
structField := newAddStructFieldSchema("profile")
task := &addCollectionStructFieldTask{
Condition: NewTaskCondition(ctx),
AddCollectionStructFieldRequest: &milvuspb.AddCollectionStructFieldRequest{
DbName: "",
CollectionName: oldSchema.GetName(),
StructArrayFieldSchema: structField,
},
ctx: ctx,
mixCoord: NewMixCoordMock(),
oldSchema: oldSchema,
}
require.NoError(t, task.OnEnqueue())
assert.Equal(t, commonpb.MsgType_AddCollectionField, task.Type())
require.NoError(t, task.PreExecute(ctx))
got := task.GetStructArrayFieldSchema()
require.NotSame(t, structField, got)
require.Len(t, got.GetFields(), 2)
assert.Equal(t, "profile[ints]", got.GetFields()[0].GetName())
assert.Equal(t, "profile[vectors]", got.GetFields()[1].GetName())
assert.True(t, got.GetFields()[0].GetNullable())
assert.True(t, got.GetFields()[1].GetNullable())
// PreExecute works on a cloned schema, so callers do not observe partial mutation on failure.
assert.Equal(t, "ints", structField.GetFields()[0].GetName())
assert.Equal(t, "vectors", structField.GetFields()[1].GetName())
}
func TestValidateAddStructFieldRequest(t *testing.T) {
baseSchema := &schemapb.CollectionSchema{
Name: "test_collection",
Fields: []*schemapb.FieldSchema{
{
Name: "pk",
DataType: schemapb.DataType_Int64,
},
},
}
t.Run("valid", func(t *testing.T) {
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("profile"))
require.NoError(t, err)
})
t.Run("struct must be nullable", func(t *testing.T) {
structField := newAddStructFieldSchema("profile")
structField.Nullable = false
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), structField)
require.Error(t, err)
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
assert.Contains(t, err.Error(), "added struct field must be nullable")
})
t.Run("duplicate parent name", func(t *testing.T) {
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("pk"))
require.Error(t, err)
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
assert.Contains(t, err.Error(), "duplicated field name pk")
})
t.Run("duplicate transformed sub field name", func(t *testing.T) {
schema := proto.Clone(baseSchema).(*schemapb.CollectionSchema)
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{
Name: "profile[ints]",
DataType: schemapb.DataType_Int64,
})
err := validateAddStructFieldRequest(schema, newAddStructFieldSchema("profile"))
require.Error(t, err)
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
assert.Contains(t, err.Error(), "duplicated field name profile[ints]")
})
t.Run("max field count includes struct parent", func(t *testing.T) {
Params.Save(Params.ProxyCfg.MaxFieldNum.Key, "3")
defer Params.Reset(Params.ProxyCfg.MaxFieldNum.Key)
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("profile"))
require.Error(t, err)
assert.Contains(t, err.Error(), "maximum field's number should be limited to 3")
})
t.Run("vector count includes array of vector sub field", func(t *testing.T) {
Params.Save(Params.ProxyCfg.MaxVectorFieldNum.Key, "0")
defer Params.Reset(Params.ProxyCfg.MaxVectorFieldNum.Key)
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("profile"))
require.Error(t, err)
assert.Contains(t, err.Error(), "maximum vector field's number should be limited to 0")
})
}
func newAddStructFieldSchema(name string) *schemapb.StructArrayFieldSchema {
return &schemapb.StructArrayFieldSchema{
Name: name,
Nullable: true,
Fields: []*schemapb.FieldSchema{
{
Name: "ints",
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int64,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.MaxCapacityKey, Value: "16"},
},
},
{
Name: "vectors",
DataType: schemapb.DataType_ArrayOfVector,
ElementType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.DimKey, Value: "8"},
{Key: common.MaxCapacityKey, Value: "16"},
},
},
},
}
}
func TestCreateCollectionTask(t *testing.T) {
mix := NewMixCoordMock()
ctx := context.Background()
+1
View File
@@ -28,6 +28,7 @@ const (
globalIDAllocatorSubPath = "gid"
globalTSOAllocatorKey = "timestamp"
globalTSOAllocatorSubPath = "tso"
defaultMaxArrayCapacity = 4096
)
func checkGeneralCapacity(ctx context.Context, newColNum int,
@@ -52,12 +52,21 @@ func (c *Core) broadcastAlterCollectionForAddField(ctx context.Context, req *mil
}
// check if the field already exists
fieldNames := typeutil.NewSet[string]()
for _, field := range coll.Fields {
if field.Name == fieldSchema.Name {
// TODO: idempotency check here.
return merr.WrapErrParameterInvalidMsg("field already exists, name: %s", fieldSchema.Name)
fieldNames.Insert(field.Name)
}
for _, structField := range coll.StructArrayFields {
fieldNames.Insert(structField.Name)
for _, field := range structField.Fields {
fieldNames.Insert(field.Name)
fieldNames.Insert(storedRootStructSubFieldName(structField.Name, field.Name))
}
}
if fieldNames.Contain(fieldSchema.Name) {
// TODO: idempotency check here.
return merr.WrapErrParameterInvalidMsg("field already exists, name: %s", fieldSchema.Name)
}
// assign a new field id.
fieldSchema.FieldID = nextFieldID(coll)
@@ -0,0 +1,271 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rootcoord
import (
"context"
"fmt"
"strings"
"github.com/cockroachdb/errors"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/fieldmaskpb"
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/distributed/streaming"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
// broadcastAlterCollectionForAddStructField broadcasts the put collection message for add struct field.
func (c *Core) broadcastAlterCollectionForAddStructField(ctx context.Context, req *milvuspb.AddCollectionStructFieldRequest) error {
broadcaster, err := c.startBroadcastWithAliasOrCollectionLock(ctx, req.GetDbName(), req.GetCollectionName())
if err != nil {
return err
}
defer broadcaster.Close()
coll, err := c.meta.GetCollectionByName(ctx, req.GetDbName(), req.GetCollectionName(), typeutil.MaxTimestamp, false)
if err != nil {
return err
}
if req.GetStructArrayFieldSchema() == nil {
return merr.WrapErrParameterInvalidMsg("struct array field schema is nil")
}
structArrayField := proto.Clone(req.GetStructArrayFieldSchema()).(*schemapb.StructArrayFieldSchema)
if err := normalizeAndCheckAddedStructField(structArrayField); err != nil {
return err
}
if err := checkStructArrayFieldSchema([]*schemapb.StructArrayFieldSchema{structArrayField}); err != nil {
return errors.Wrap(err, "failed to check struct array field schema")
}
if err := validateStructArrayFieldDataType([]*schemapb.StructArrayFieldSchema{structArrayField}); err != nil {
return err
}
if err := checkAddStructFieldDuplicate(coll, structArrayField); err != nil {
return err
}
fieldIDStart := nextFieldID(coll)
structArrayField.FieldID = fieldIDStart
for i, field := range structArrayField.GetFields() {
field.FieldID = fieldIDStart + int64(i) + 1
}
schema := coll.ToCollectionSchemaPB()
schema.Version = coll.SchemaVersion + 1
schema.StructArrayFields = append(schema.StructArrayFields, structArrayField)
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
if err != nil {
return err
}
channels := make([]string, 0, len(coll.VirtualChannelNames)+1)
channels = append(channels, streaming.WAL().ControlChannel())
channels = append(channels, coll.VirtualChannelNames...)
msg := message.NewAlterCollectionMessageBuilderV2().
WithHeader(&messagespb.AlterCollectionMessageHeader{
DbId: coll.DBID,
CollectionId: coll.CollectionID,
UpdateMask: &fieldmaskpb.FieldMask{
Paths: []string{message.FieldMaskCollectionSchema},
},
CacheExpirations: cacheExpirations,
}).
WithBody(&messagespb.AlterCollectionMessageBody{
Updates: &messagespb.AlterCollectionMessageUpdates{
Schema: schema,
},
}).
WithBroadcast(channels).
MustBuildBroadcast()
if _, err := broadcaster.Broadcast(ctx, msg); err != nil {
return err
}
return nil
}
func normalizeAndCheckAddedStructField(structArrayField *schemapb.StructArrayFieldSchema) error {
if !structArrayField.GetNullable() {
return merr.WrapErrParameterInvalidMsg("added struct field must be nullable, please check it, struct field name = %s", structArrayField.GetName())
}
if err := validateAddedStructFieldName(structArrayField.GetName()); err != nil {
return err
}
if isReservedStructFieldName(structArrayField.GetName()) {
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", structArrayField.GetName())
}
for _, field := range structArrayField.GetFields() {
originalName, err := normalizeStructSubFieldName(structArrayField.GetName(), field)
if err != nil {
return err
}
if err := validateAddedStructFieldName(originalName); err != nil {
return err
}
if isReservedStructFieldName(originalName) {
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", originalName)
}
if field.GetIsPrimaryKey() {
return merr.WrapErrParameterInvalidMsg("primary key is not supported for struct field, field name = %s", originalName)
}
if field.GetAutoID() {
return merr.WrapErrParameterInvalidMsg("autoID is not supported for struct field, field name = %s", originalName)
}
if field.GetIsPartitionKey() {
return merr.WrapErrParameterInvalidMsg("partition key is not supported for struct field, field name = %s", originalName)
}
if field.GetIsClusteringKey() {
return merr.WrapErrParameterInvalidMsg("clustering key is not supported for struct field, field name = %s", originalName)
}
if field.GetDefaultValue() != nil {
return merr.WrapErrParameterInvalidMsg("default value is not supported for struct field, field name = %s", originalName)
}
if field.GetIsFunctionOutput() {
return merr.WrapErrParameterInvalidMsg("function output is not supported for struct field, field name = %s", originalName)
}
if field.GetExternalField() != "" {
return merr.WrapErrParameterInvalidMsg("add struct field operation does not support external field mapping, field name = %s", originalName)
}
field.Nullable = true
}
return nil
}
func normalizeStructSubFieldName(structName string, field *schemapb.FieldSchema) (string, error) {
fieldName := field.GetName()
if isStoredStructSubFieldName(structName, fieldName) {
originalName, err := typeutil.ExtractStructFieldName(fieldName)
if err != nil {
return "", err
}
return originalName, nil
}
if typeutil.IsStructSubField(fieldName) {
return "", merr.WrapErrParameterInvalidMsg("invalid struct sub-field name %s for struct field %s", fieldName, structName)
}
field.Name = typeutil.ConcatStructFieldName(structName, fieldName)
return fieldName, nil
}
func checkAddStructFieldDuplicate(coll *model.Collection, structArrayField *schemapb.StructArrayFieldSchema) error {
fieldNames := make(map[string]struct{})
addFieldName := func(name string) error {
if _, ok := fieldNames[name]; ok {
return merr.WrapErrParameterInvalidMsg("field already exists, name: %s", name)
}
fieldNames[name] = struct{}{}
return nil
}
for _, field := range coll.Fields {
if err := addFieldName(field.Name); err != nil {
return err
}
}
for _, structField := range coll.StructArrayFields {
if err := addFieldName(structField.Name); err != nil {
return err
}
for _, field := range structField.Fields {
if err := addFieldName(field.Name); err != nil {
return err
}
storedName := storedRootStructSubFieldName(structField.Name, field.Name)
if storedName != field.Name {
if err := addFieldName(storedName); err != nil {
return err
}
}
}
}
if err := addFieldName(structArrayField.GetName()); err != nil {
return err
}
for _, field := range structArrayField.GetFields() {
if err := addFieldName(field.GetName()); err != nil {
return err
}
}
return nil
}
func isStoredStructSubFieldName(structName string, fieldName string) bool {
return strings.HasPrefix(fieldName, structName+"[") && strings.HasSuffix(fieldName, "]")
}
func storedRootStructSubFieldName(structName string, fieldName string) string {
if isStoredStructSubFieldName(structName, fieldName) {
return fieldName
}
return typeutil.ConcatStructFieldName(structName, fieldName)
}
func isReservedStructFieldName(fieldName string) bool {
return funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, fieldName)
}
func validateAddedStructFieldName(fieldName string) error {
fieldName = strings.TrimSpace(fieldName)
if fieldName == "" {
return merr.WrapErrFieldNameInvalid(fieldName, "field name should not be empty")
}
invalidMsg := "Invalid field name: " + fieldName + ". "
if len(fieldName) > Params.ProxyCfg.MaxNameLength.GetAsInt() {
msg := invalidMsg + "The length of a field name must be less than " + Params.ProxyCfg.MaxNameLength.GetValue() + " characters."
return merr.WrapErrFieldNameInvalid(fieldName, msg)
}
firstChar := fieldName[0]
if firstChar != '_' && !isAlphaForStructFieldName(firstChar) {
msg := invalidMsg + "The first character of a field name must be an underscore or letter."
return merr.WrapErrFieldNameInvalid(fieldName, msg)
}
for i := 1; i < len(fieldName); i++ {
c := fieldName[i]
if c != '_' && !isAlphaForStructFieldName(c) && !isNumberForStructFieldName(c) {
msg := invalidMsg + "Field name can only contain numbers, letters, and underscores."
return merr.WrapErrFieldNameInvalid(fieldName, msg)
}
}
if _, ok := common.FieldNameKeywords[fieldName]; ok {
msg := invalidMsg + fmt.Sprintf("%s is keyword in milvus.", fieldName)
return merr.WrapErrFieldNameInvalid(fieldName, msg)
}
return nil
}
func isAlphaForStructFieldName(c uint8) bool {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
}
func isNumberForStructFieldName(c uint8) bool {
return c >= '0' && c <= '9'
}
@@ -0,0 +1,129 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rootcoord
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
func TestDDLCallbacksAlterCollectionAddStructField(t *testing.T) {
core := initStreamingSystemAndCore(t)
ctx := context.Background()
dbName := "testDB" + funcutil.RandomString(10)
collectionName := "testCollection" + funcutil.RandomString(10)
createCollectionForTest(t, ctx, core, dbName, collectionName)
resp, err := core.AddCollectionStructField(ctx, &milvuspb.AddCollectionStructFieldRequest{
DbName: dbName,
CollectionName: collectionName,
StructArrayFieldSchema: newRootAddStructFieldSchema("field1"),
})
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
resp, err = core.AddCollectionStructField(ctx, &milvuspb.AddCollectionStructFieldRequest{
DbName: dbName,
CollectionName: collectionName,
StructArrayFieldSchema: newRootAddStructFieldSchema("profile"),
})
require.NoError(t, merr.CheckRPCCall(resp, err))
coll, err := core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
require.NoError(t, err)
require.Len(t, getUserFields(coll.Fields), 1)
require.Len(t, coll.StructArrayFields, 1)
structField := coll.StructArrayFields[0]
require.Equal(t, int64(101), structField.FieldID)
require.Equal(t, "profile", structField.Name)
require.True(t, structField.Nullable)
require.Len(t, structField.Fields, 2)
require.Equal(t, int64(102), structField.Fields[0].FieldID)
require.Equal(t, "profile[ints]", structField.Fields[0].Name)
require.Equal(t, schemapb.DataType_Array, structField.Fields[0].DataType)
require.True(t, structField.Fields[0].Nullable)
require.Equal(t, int64(103), structField.Fields[1].FieldID)
require.Equal(t, "profile[vectors]", structField.Fields[1].Name)
require.Equal(t, schemapb.DataType_ArrayOfVector, structField.Fields[1].DataType)
require.True(t, structField.Fields[1].Nullable)
assertSchemaVersion(t, ctx, core, dbName, collectionName, 1)
resp, err = core.AddCollectionStructField(ctx, &milvuspb.AddCollectionStructFieldRequest{
DbName: dbName,
CollectionName: collectionName,
StructArrayFieldSchema: newRootAddStructFieldSchema("profile"),
})
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
resp, err = core.AddCollectionField(ctx, &milvuspb.AddCollectionFieldRequest{
DbName: dbName,
CollectionName: collectionName,
Schema: getFieldSchema("profile"),
})
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
}
func getUserFields(fields []*model.Field) []*model.Field {
ret := make([]*model.Field, 0, len(fields))
for _, field := range fields {
if field.Name == RowIDFieldName || field.Name == TimeStampFieldName {
continue
}
ret = append(ret, field)
}
return ret
}
func newRootAddStructFieldSchema(name string) *schemapb.StructArrayFieldSchema {
return &schemapb.StructArrayFieldSchema{
Name: name,
Nullable: true,
Fields: []*schemapb.FieldSchema{
{
Name: typeutil.ConcatStructFieldName(name, "ints"),
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int64,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.MaxCapacityKey, Value: "16"},
},
},
{
Name: typeutil.ConcatStructFieldName(name, "vectors"),
DataType: schemapb.DataType_ArrayOfVector,
ElementType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.DimKey, Value: "8"},
{Key: common.MaxCapacityKey, Value: "16"},
},
},
},
}
}
+26
View File
@@ -1025,6 +1025,32 @@ func (c *Core) AddCollectionField(ctx context.Context, in *milvuspb.AddCollectio
return merr.Success(), nil
}
// AddCollectionStructField add struct field
func (c *Core) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
if err := merr.CheckHealthy(c.GetStateCode()); err != nil {
return merr.Status(err), nil
}
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionStructField", metrics.TotalLabel).Inc()
tr := timerecord.NewTimeRecorder("AddCollectionStructField")
log := log.Ctx(ctx).With(zap.String("role", typeutil.RootCoordRole),
zap.String("dbName", in.GetDbName()),
zap.String("collectionName", in.GetCollectionName()))
log.Info("received request to add collection struct field")
if err := c.broadcastAlterCollectionForAddStructField(ctx, in); err != nil {
log.Info("failed to add collection struct field", zap.Error(err))
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionStructField", metrics.FailLabel).Inc()
return merr.Status(err), nil
}
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionStructField", metrics.SuccessLabel).Inc()
metrics.RootCoordDDLReqLatency.WithLabelValues("AddCollectionStructField").Observe(float64(tr.ElapseSpan().Milliseconds()))
log.Info("done to add collection struct field")
return merr.Success(), nil
}
func (c *Core) AlterCollectionSchema(ctx context.Context, in *milvuspb.AlterCollectionSchemaRequest) (*milvuspb.AlterCollectionSchemaResponse, error) {
if err := merr.CheckHealthy(c.GetStateCode()); err != nil {
return &milvuspb.AlterCollectionSchemaResponse{
+3
View File
@@ -507,6 +507,9 @@ func getStructSubFieldMaxCapacity(structName string, field *schemapb.FieldSchema
return 0, merr.WrapErrParameterInvalidMsg("the value for %s of field %s in struct array field %s must be an integer",
common.MaxCapacityKey, field.GetName(), structName)
}
if maxCapacity > defaultMaxArrayCapacity || maxCapacity <= 0 {
return 0, merr.WrapErrParameterInvalidMsg("the maximum capacity specified for a Array should be in (0, %d]", defaultMaxArrayCapacity)
}
return maxCapacity, nil
}
return 0, merr.WrapErrParameterInvalidMsg("type param(%s) should be specified for field %s in struct array field %s",
+1 -1
View File
@@ -15,7 +15,7 @@ import (
)
func ConvertToArrowSchema(schema *schemapb.CollectionSchema, useFieldID bool) (*arrow.Schema, error) {
fieldCount := typeutil.GetTotalFieldsNum(schema)
fieldCount := len(typeutil.GetAllFieldSchemas(schema))
arrowFields := make([]arrow.Field, 0, fieldCount)
appendArrowField := func(field *schemapb.FieldSchema) error {
if serdeMap[field.DataType].arrowType == nil {
@@ -93,6 +93,10 @@ func (m *GrpcRootCoordClient) AddCollectionField(ctx context.Context, in *milvus
return &commonpb.Status{}, m.Err
}
func (m *GrpcRootCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return &commonpb.Status{}, m.Err
}
func (m *GrpcRootCoordClient) ListPolicy(ctx context.Context, in *internalpb.ListPolicyRequest, opts ...grpc.CallOption) (*internalpb.ListPolicyResponse, error) {
return &internalpb.ListPolicyResponse{}, m.Err
}
+1 -1
View File
@@ -22,7 +22,7 @@ require (
github.com/jolestar/go-commons-pool/v2 v2.1.2
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12
github.com/klauspost/compress v1.18.0
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
github.com/minio/minio-go/v7 v7.0.73
github.com/panjf2000/ants/v2 v2.11.3
github.com/prometheus/client_golang v1.20.5
+2 -2
View File
@@ -517,8 +517,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo=
+9
View File
@@ -49,6 +49,15 @@ service RootCoord {
*/
rpc AddCollectionField(milvus.AddCollectionFieldRequest) returns (common.Status) {}
/**
* @brief This method is used to add collection struct field.
*
* @param AddCollectionStructFieldRequest, struct field schema is going to be added.
*
* @return Status
*/
rpc AddCollectionStructField(milvus.AddCollectionStructFieldRequest) returns (common.Status) {}
/**
* @brief This method is used to add/drop collection field/function.
File diff suppressed because it is too large Load Diff
@@ -30,6 +30,7 @@ const (
RootCoord_DropCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/DropCollection"
RootCoord_TruncateCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/TruncateCollection"
RootCoord_AddCollectionField_FullMethodName = "/milvus.proto.rootcoord.RootCoord/AddCollectionField"
RootCoord_AddCollectionStructField_FullMethodName = "/milvus.proto.rootcoord.RootCoord/AddCollectionStructField"
RootCoord_AlterCollectionSchema_FullMethodName = "/milvus.proto.rootcoord.RootCoord/AlterCollectionSchema"
RootCoord_HasCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/HasCollection"
RootCoord_DescribeCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/DescribeCollection"
@@ -133,6 +134,13 @@ type RootCoordClient interface {
// @return Status
AddCollectionField(ctx context.Context, in *milvuspb.AddCollectionFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
// @brief This method is used to add collection struct field.
//
// @param AddCollectionStructFieldRequest, struct field schema is going to be added.
//
// @return Status
AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
// @brief This method is used to add/drop collection field/function.
//
// @param AlterCollectionSchemaRequest, field schema is going to be added.
@@ -316,6 +324,15 @@ func (c *rootCoordClient) AddCollectionField(ctx context.Context, in *milvuspb.A
return out, nil
}
func (c *rootCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, RootCoord_AddCollectionStructField_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *rootCoordClient) AlterCollectionSchema(ctx context.Context, in *milvuspb.AlterCollectionSchemaRequest, opts ...grpc.CallOption) (*milvuspb.AlterCollectionSchemaResponse, error) {
out := new(milvuspb.AlterCollectionSchemaResponse)
err := c.cc.Invoke(ctx, RootCoord_AlterCollectionSchema_FullMethodName, in, out, opts...)
@@ -937,6 +954,13 @@ type RootCoordServer interface {
// @return Status
AddCollectionField(context.Context, *milvuspb.AddCollectionFieldRequest) (*commonpb.Status, error)
// *
// @brief This method is used to add collection struct field.
//
// @param AddCollectionStructFieldRequest, struct field schema is going to be added.
//
// @return Status
AddCollectionStructField(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)
// *
// @brief This method is used to add/drop collection field/function.
//
// @param AlterCollectionSchemaRequest, field schema is going to be added.
@@ -1074,6 +1098,9 @@ func (UnimplementedRootCoordServer) TruncateCollection(context.Context, *milvusp
func (UnimplementedRootCoordServer) AddCollectionField(context.Context, *milvuspb.AddCollectionFieldRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method AddCollectionField not implemented")
}
func (UnimplementedRootCoordServer) AddCollectionStructField(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method AddCollectionStructField not implemented")
}
func (UnimplementedRootCoordServer) AlterCollectionSchema(context.Context, *milvuspb.AlterCollectionSchemaRequest) (*milvuspb.AlterCollectionSchemaResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method AlterCollectionSchema not implemented")
}
@@ -1407,6 +1434,24 @@ func _RootCoord_AddCollectionField_Handler(srv interface{}, ctx context.Context,
return interceptor(ctx, in, info, handler)
}
func _RootCoord_AddCollectionStructField_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(milvuspb.AddCollectionStructFieldRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(RootCoordServer).AddCollectionStructField(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: RootCoord_AddCollectionStructField_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(RootCoordServer).AddCollectionStructField(ctx, req.(*milvuspb.AddCollectionStructFieldRequest))
}
return interceptor(ctx, in, info, handler)
}
func _RootCoord_AlterCollectionSchema_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(milvuspb.AlterCollectionSchemaRequest)
if err := dec(in); err != nil {
@@ -2612,6 +2657,10 @@ var RootCoord_ServiceDesc = grpc.ServiceDesc{
MethodName: "AddCollectionField",
Handler: _RootCoord_AddCollectionField_Handler,
},
{
MethodName: "AddCollectionStructField",
Handler: _RootCoord_AddCollectionStructField_Handler,
},
{
MethodName: "AlterCollectionSchema",
Handler: _RootCoord_AlterCollectionSchema_Handler,
+3 -3
View File
@@ -2197,12 +2197,12 @@ func MergeFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData) error
return nil
}
// GetTotalFieldsNum get total fields number
// We exclude StructArrayField itself as it does not contain data directly.
// GetTotalFieldsNum get total fields number, including StructArrayField itself.
func GetTotalFieldsNum(schema *schemapb.CollectionSchema) int {
num := len(schema.GetFields())
for _, structArrayField := range schema.GetStructArrayFields() {
num += len(structArrayField.GetFields())
// +1 for the StructArrayField itself
num += len(structArrayField.GetFields()) + 1
}
return num
}
+21
View File
@@ -5974,3 +5974,24 @@ func TestAppendFieldData_ArrayOfVectorNullRowAppendsPlaceholder(t *testing.T) {
assert.EqualValues(t, 1, got.GetDim())
assert.Equal(t, schemapb.DataType_FloatVector, got.GetElementType())
}
func TestGetTotalFieldsNumIncludesStructParent(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "pk", DataType: schemapb.DataType_Int64},
{Name: "scalar", DataType: schemapb.DataType_Bool},
},
StructArrayFields: []*schemapb.StructArrayFieldSchema{
{
Name: "profile",
Fields: []*schemapb.FieldSchema{
{Name: "profile[ints]", DataType: schemapb.DataType_Array},
{Name: "profile[vectors]", DataType: schemapb.DataType_ArrayOfVector},
},
},
},
}
assert.Equal(t, 5, GetTotalFieldsNum(schema))
assert.Len(t, GetAllFieldSchemas(schema), 4)
}
+1 -1
View File
@@ -4,7 +4,7 @@ go 1.25.9
require (
github.com/apache/arrow/go/v17 v17.0.0
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
github.com/milvus-io/milvus/client/v2 v2.0.0-20241125024034-0b9edb62a92d
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
github.com/minio/minio-go/v7 v7.0.73
+2 -2
View File
@@ -245,8 +245,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=