mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 02:05:41 +00:00
feat: impl StructArray -- support dynamic add struct field (#49807)
issue: https://github.com/milvus-io/milvus/issues/42148 design doc: docs/design-docs/design_docs/20260306-struct.md this PR also fixes https://github.com/milvus-io/milvus/issues/49693 --------- Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
This commit is contained in:
+3
-3
@@ -7,15 +7,13 @@ require (
|
||||
github.com/cockroachdb/errors v1.9.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
|
||||
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.23
|
||||
github.com/samber/lo v1.52.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tidwall/gjson v1.17.1
|
||||
go.opentelemetry.io/otel v1.43.0
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
@@ -95,6 +93,7 @@ require (
|
||||
go.etcd.io/etcd/server/v3 v3.5.23 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
@@ -105,6 +104,7 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/crypto v0.49.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
|
||||
+2
-4
@@ -222,10 +222,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
|
||||
@@ -86,6 +86,65 @@ func (_c *MilvusServiceServer_AddCollectionField_Call) RunAndReturn(run func(con
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MilvusServiceServer) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AddCollectionStructField")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MilvusServiceServer_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
|
||||
type MilvusServiceServer_AddCollectionStructField_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AddCollectionStructField is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *milvuspb.AddCollectionStructFieldRequest
|
||||
func (_e *MilvusServiceServer_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MilvusServiceServer_AddCollectionStructField_Call {
|
||||
return &MilvusServiceServer_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MilvusServiceServer_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MilvusServiceServer_AddCollectionStructField_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MilvusServiceServer_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MilvusServiceServer_AddCollectionStructField_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MilvusServiceServer_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MilvusServiceServer_AddCollectionStructField_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MilvusServiceServer) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@@ -69,7 +69,7 @@ require (
|
||||
github.com/hamba/avro/v2 v2.29.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/magiconair/properties v1.8.7
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
|
||||
github.com/milvus-io/milvus/client/v2 v2.6.2
|
||||
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
|
||||
github.com/shirou/gopsutil/v4 v4.25.10
|
||||
|
||||
@@ -814,10 +814,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
|
||||
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
||||
|
||||
@@ -575,6 +575,10 @@ func (c *mixCoordImpl) AddCollectionField(ctx context.Context, in *milvuspb.AddC
|
||||
return c.rootcoordServer.AddCollectionField(ctx, in)
|
||||
}
|
||||
|
||||
func (c *mixCoordImpl) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
return c.rootcoordServer.AddCollectionStructField(ctx, in)
|
||||
}
|
||||
|
||||
func (s *mixCoordImpl) CreateCredential(ctx context.Context, req *internalpb.CredentialInfo) (*commonpb.Status, error) {
|
||||
return s.rootcoordServer.CreateCredential(ctx, req)
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@
|
||||
#include "index/VectorDiskIndex.h"
|
||||
#include "index/VectorMemIndex.h"
|
||||
#include "knowhere/comp/knowhere_check.h"
|
||||
#include "knowhere/emb_list_utils.h"
|
||||
#include "knowhere/expected.h"
|
||||
#include "knowhere/index/index_static.h"
|
||||
#include "knowhere/operands.h"
|
||||
@@ -256,6 +257,9 @@ IndexFactory::VecIndexLoadResource(
|
||||
index_type, index_version, config);
|
||||
break;
|
||||
case milvus::DataType::VECTOR_ARRAY: {
|
||||
auto metric_type = milvus::index::GetMetricTypeFromConfig(config);
|
||||
auto is_emb_list_metric =
|
||||
knowhere::get_el_metric_type(metric_type).has_value();
|
||||
switch (element_type) {
|
||||
case milvus::DataType::VECTOR_FLOAT:
|
||||
resource = knowhere::IndexStaticFaced<knowhere::fp32>::
|
||||
@@ -326,6 +330,11 @@ IndexFactory::VecIndexLoadResource(
|
||||
element_type);
|
||||
return LoadResourceRequest{0, 0, 0, 0, true};
|
||||
}
|
||||
// Non-emb-list VECTOR_ARRAY indexes do not keep embedding-list
|
||||
// offsets, so they cannot reconstruct row-level embedding lists.
|
||||
if (!is_emb_list_metric) {
|
||||
has_raw_data = false;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -3418,12 +3418,31 @@ ChunkedSegmentSealedImpl::get_raw_data(milvus::OpContext* op_ctx,
|
||||
break;
|
||||
}
|
||||
case DataType::VECTOR_ARRAY: {
|
||||
bulk_subscript_vector_array_impl(
|
||||
op_ctx,
|
||||
column.get(),
|
||||
valid_offsets,
|
||||
valid_count,
|
||||
ret->mutable_vectors()->mutable_vector_array()->mutable_data());
|
||||
auto dst =
|
||||
ret->mutable_vectors()->mutable_vector_array()->mutable_data();
|
||||
if (field_meta.is_nullable() && valid_data != nullptr) {
|
||||
if (valid_count == 0) {
|
||||
break;
|
||||
}
|
||||
std::vector<int64_t> valid_logical_offsets;
|
||||
valid_logical_offsets.reserve(valid_count);
|
||||
for (int64_t i = 0; i < count; ++i) {
|
||||
if (valid_data[i]) {
|
||||
valid_logical_offsets.push_back(i);
|
||||
}
|
||||
}
|
||||
column->BulkVectorArrayAt(
|
||||
op_ctx,
|
||||
[dst, &valid_logical_offsets](VectorFieldProto&& array,
|
||||
size_t i) {
|
||||
dst->at(valid_logical_offsets[i]) = std::move(array);
|
||||
},
|
||||
valid_offsets,
|
||||
valid_count);
|
||||
} else {
|
||||
bulk_subscript_vector_array_impl(
|
||||
op_ctx, column.get(), seg_offsets, count, dst);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
|
||||
@@ -774,7 +774,8 @@ SegmentInternalInterface::bulk_subscript_not_exist_field(
|
||||
AssertInfo(field_meta.is_nullable(),
|
||||
"Non-nullable vector field should not reach here");
|
||||
|
||||
auto result = CreateEmptyVectorDataArray(0, field_meta);
|
||||
auto create_count = IsVectorArrayDataType(data_type) ? count : 0;
|
||||
auto result = CreateEmptyVectorDataArray(create_count, field_meta);
|
||||
|
||||
auto valid_data = result->mutable_valid_data();
|
||||
for (int64_t i = 0; i < count; ++i) {
|
||||
|
||||
@@ -501,6 +501,7 @@ CreateEmptyVectorDataArray(int64_t count, const FieldMeta& field_meta) {
|
||||
}
|
||||
case DataType::VECTOR_ARRAY: {
|
||||
auto obj = vector_array->mutable_vector_array();
|
||||
obj->set_dim(dim);
|
||||
obj->set_element_type(static_cast<milvus::proto::schema::DataType>(
|
||||
field_meta.get_element_type()));
|
||||
obj->mutable_data()->Reserve(count);
|
||||
@@ -523,7 +524,8 @@ CreateEmptyVectorDataArray(int64_t count,
|
||||
int64_t valid_count,
|
||||
const void* valid_data,
|
||||
const FieldMeta& field_meta) {
|
||||
int64_t data_count = (field_meta.is_nullable() && valid_data != nullptr)
|
||||
int64_t data_count = (field_meta.is_nullable() && valid_data != nullptr &&
|
||||
field_meta.get_data_type() != DataType::VECTOR_ARRAY)
|
||||
? valid_count
|
||||
: count;
|
||||
auto data_array = CreateEmptyVectorDataArray(data_count, field_meta);
|
||||
|
||||
@@ -184,6 +184,12 @@ DefaultValueChunkTranslator::value_size() const {
|
||||
case milvus::DataType::ARRAY:
|
||||
value_size = sizeof(Array);
|
||||
break;
|
||||
case milvus::DataType::VECTOR_ARRAY:
|
||||
AssertInfo(field_meta_.is_nullable(),
|
||||
"only nullable vector array fields can be "
|
||||
"dynamically added");
|
||||
value_size = 0;
|
||||
break;
|
||||
case milvus::DataType::VECTOR_FLOAT:
|
||||
case milvus::DataType::VECTOR_BINARY:
|
||||
case milvus::DataType::VECTOR_FLOAT16:
|
||||
@@ -231,7 +237,16 @@ DefaultValueChunkTranslator::build_buffer_for_rows(
|
||||
auto data_type = field_meta_.get_data_type();
|
||||
std::shared_ptr<arrow::ArrayBuilder> builder;
|
||||
|
||||
if (IsVectorDataType(data_type)) {
|
||||
if (data_type == milvus::DataType::VECTOR_ARRAY) {
|
||||
AssertInfo(field_meta_.is_nullable(),
|
||||
"only nullable vector array fields can be "
|
||||
"dynamically added");
|
||||
builder =
|
||||
milvus::storage::CreateArrowBuilder(data_type,
|
||||
field_meta_.get_element_type(),
|
||||
field_meta_.get_dim(),
|
||||
true);
|
||||
} else if (IsVectorDataType(data_type)) {
|
||||
AssertInfo(field_meta_.is_nullable(),
|
||||
"only nullable vector fields can be dynamically added");
|
||||
builder = std::make_shared<arrow::BinaryBuilder>();
|
||||
|
||||
@@ -1091,6 +1091,49 @@ TEST_P(DefaultValueChunkTranslatorTest, TestNullableVectorFloat) {
|
||||
EXPECT_EQ(total_rows, row_count);
|
||||
}
|
||||
|
||||
// Test nullable VECTOR_ARRAY with all-null default values.
|
||||
TEST_P(DefaultValueChunkTranslatorTest, TestNullableVectorArray) {
|
||||
int64_t dim = 4;
|
||||
int64_t row_count = 100;
|
||||
|
||||
FieldMeta field_meta(FieldName("test_vector_array_nullable"),
|
||||
FieldId(1302),
|
||||
DataType::VECTOR_ARRAY,
|
||||
DataType::VECTOR_FLOAT,
|
||||
dim,
|
||||
std::nullopt, // metric_type
|
||||
true); // nullable
|
||||
|
||||
FieldDataInfo field_data_info(1302, row_count, getMmapDirPath());
|
||||
|
||||
auto translator = std::make_unique<DefaultValueChunkTranslator>(
|
||||
segment_id_, field_meta, field_data_info, GetParam(), true);
|
||||
|
||||
EXPECT_EQ(translator->value_size(), 0);
|
||||
|
||||
size_t num_cells = translator->num_cells();
|
||||
ASSERT_GE(num_cells, 1);
|
||||
|
||||
std::vector<cachinglayer::cid_t> cids;
|
||||
for (size_t i = 0; i < num_cells; ++i) {
|
||||
cids.push_back(i);
|
||||
}
|
||||
|
||||
auto cells = translator->get_cells(nullptr, cids);
|
||||
EXPECT_EQ(cells.size(), num_cells);
|
||||
|
||||
int64_t total_rows = 0;
|
||||
for (auto& [cid, chunk] : cells) {
|
||||
total_rows += chunk->RowNums();
|
||||
for (int64_t i = 0; i < chunk->RowNums(); ++i) {
|
||||
EXPECT_FALSE(chunk->isValid(i))
|
||||
<< "Expected null at cell " << cid << " row " << i;
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(total_rows, row_count);
|
||||
}
|
||||
|
||||
// Test nullable VECTOR_SPARSE_U32_F32 with all-null default values
|
||||
TEST_P(DefaultValueChunkTranslatorTest, TestNullableSparseVector) {
|
||||
bool use_mmap = GetParam();
|
||||
|
||||
@@ -220,6 +220,10 @@ func (m *mockMixCoord) AddCollectionField(ctx context.Context, req *milvuspb.Add
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (m *mockMixCoord) AddCollectionStructField(ctx context.Context, req *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (m *mockMixCoord) GetQuotaMetrics(ctx context.Context, req *internalpb.GetQuotaMetricsRequest) (*internalpb.GetQuotaMetricsResponse, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
@@ -259,6 +259,17 @@ func (c *Client) AddCollectionField(ctx context.Context, in *milvuspb.AddCollect
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Client) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
in = typeutil.Clone(in)
|
||||
commonpbutil.UpdateMsgBase(
|
||||
in.GetBase(),
|
||||
commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())),
|
||||
)
|
||||
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*commonpb.Status, error) {
|
||||
return client.AddCollectionStructField(ctx, in)
|
||||
})
|
||||
}
|
||||
|
||||
// DescribeCollection return collection info
|
||||
func (c *Client) DescribeCollection(ctx context.Context, in *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error) {
|
||||
in = typeutil.Clone(in)
|
||||
|
||||
@@ -410,6 +410,10 @@ func (s *Server) AddCollectionField(ctx context.Context, in *milvuspb.AddCollect
|
||||
return s.mixCoord.AddCollectionField(ctx, in)
|
||||
}
|
||||
|
||||
func (s *Server) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
return s.mixCoord.AddCollectionStructField(ctx, in)
|
||||
}
|
||||
|
||||
// CreatePartition creates a partition in a collection
|
||||
func (s *Server) CreatePartition(ctx context.Context, in *milvuspb.CreatePartitionRequest) (*commonpb.Status, error) {
|
||||
return s.mixCoord.CreatePartition(ctx, in)
|
||||
|
||||
@@ -668,6 +668,11 @@ func (s *Server) AddCollectionField(ctx context.Context, request *milvuspb.AddCo
|
||||
return s.proxy.AddCollectionField(ctx, request)
|
||||
}
|
||||
|
||||
// AddCollectionStructField add a struct field to collection
|
||||
func (s *Server) AddCollectionStructField(ctx context.Context, request *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
return s.proxy.AddCollectionStructField(ctx, request)
|
||||
}
|
||||
|
||||
// GetCollectionStatistics notifies Proxy to get a collection's Statistics
|
||||
func (s *Server) GetCollectionStatistics(ctx context.Context, request *milvuspb.GetCollectionStatisticsRequest) (*milvuspb.GetCollectionStatisticsResponse, error) {
|
||||
return s.proxy.GetCollectionStatistics(ctx, request)
|
||||
|
||||
@@ -164,6 +164,65 @@ func (_c *MixCoord_AddCollectionField_Call) RunAndReturn(run func(context.Contex
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AddCollectionStructField")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MixCoord_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
|
||||
type MixCoord_AddCollectionStructField_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AddCollectionStructField is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *milvuspb.AddCollectionStructFieldRequest
|
||||
func (_e *MixCoord_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MixCoord_AddCollectionStructField_Call {
|
||||
return &MixCoord_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MixCoord_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MixCoord_AddCollectionStructField_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MixCoord_AddCollectionStructField_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MixCoord_AddCollectionStructField_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@@ -187,6 +187,80 @@ func (_c *MockMixCoordClient_AddCollectionField_Call) RunAndReturn(run func(cont
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionStructField provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AddCollectionStructField")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) *commonpb.Status); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMixCoordClient_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
|
||||
type MockMixCoordClient_AddCollectionStructField_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AddCollectionStructField is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *milvuspb.AddCollectionStructFieldRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockMixCoordClient_Expecter) AddCollectionStructField(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_AddCollectionStructField_Call {
|
||||
return &MockMixCoordClient_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_AddCollectionStructField_Call) Run(run func(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption)) *MockMixCoordClient_AddCollectionStructField_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockMixCoordClient_AddCollectionStructField_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockMixCoordClient_AddCollectionStructField_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionFunction provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) AddCollectionFunction(ctx context.Context, in *milvuspb.AddCollectionFunctionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
|
||||
@@ -92,6 +92,65 @@ func (_c *MockProxy_AddCollectionField_Call) RunAndReturn(run func(context.Conte
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockProxy) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AddCollectionStructField")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockProxy_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
|
||||
type MockProxy_AddCollectionStructField_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AddCollectionStructField is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *milvuspb.AddCollectionStructFieldRequest
|
||||
func (_e *MockProxy_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MockProxy_AddCollectionStructField_Call {
|
||||
return &MockProxy_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockProxy_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MockProxy_AddCollectionStructField_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockProxy_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockProxy_AddCollectionStructField_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockProxy_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MockProxy_AddCollectionStructField_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockProxy) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@@ -95,6 +95,65 @@ func (_c *MockRootCoord_AddCollectionField_Call) RunAndReturn(run func(context.C
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionStructField provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockRootCoord) AddCollectionStructField(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AddCollectionStructField")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockRootCoord_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
|
||||
type MockRootCoord_AddCollectionStructField_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AddCollectionStructField is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *milvuspb.AddCollectionStructFieldRequest
|
||||
func (_e *MockRootCoord_Expecter) AddCollectionStructField(_a0 interface{}, _a1 interface{}) *MockRootCoord_AddCollectionStructField_Call {
|
||||
return &MockRootCoord_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockRootCoord_AddCollectionStructField_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.AddCollectionStructFieldRequest)) *MockRootCoord_AddCollectionStructField_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockRootCoord_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockRootCoord_AddCollectionStructField_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockRootCoord_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)) *MockRootCoord_AddCollectionStructField_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionFunction provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockRootCoord) AddCollectionFunction(_a0 context.Context, _a1 *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@@ -107,6 +107,80 @@ func (_c *MockRootCoordClient_AddCollectionField_Call) RunAndReturn(run func(con
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionStructField provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockRootCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AddCollectionStructField")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) *commonpb.Status); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockRootCoordClient_AddCollectionStructField_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddCollectionStructField'
|
||||
type MockRootCoordClient_AddCollectionStructField_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AddCollectionStructField is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *milvuspb.AddCollectionStructFieldRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockRootCoordClient_Expecter) AddCollectionStructField(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AddCollectionStructField_Call {
|
||||
return &MockRootCoordClient_AddCollectionStructField_Call{Call: _e.mock.On("AddCollectionStructField",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockRootCoordClient_AddCollectionStructField_Call) Run(run func(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption)) *MockRootCoordClient_AddCollectionStructField_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*milvuspb.AddCollectionStructFieldRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockRootCoordClient_AddCollectionStructField_Call) Return(_a0 *commonpb.Status, _a1 error) *MockRootCoordClient_AddCollectionStructField_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockRootCoordClient_AddCollectionStructField_Call) RunAndReturn(run func(context.Context, *milvuspb.AddCollectionStructFieldRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockRootCoordClient_AddCollectionStructField_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AddCollectionFunction provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockRootCoordClient) AddCollectionFunction(ctx context.Context, in *milvuspb.AddCollectionFunctionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
|
||||
@@ -306,6 +306,11 @@ func fillDatabase(ctx context.Context, req interface{}) (context.Context, interf
|
||||
r.DbName = GetCurDBNameFromContextOrDefault(ctx)
|
||||
}
|
||||
return ctx, r
|
||||
case *milvuspb.AddCollectionStructFieldRequest:
|
||||
if r.DbName == "" {
|
||||
r.DbName = GetCurDBNameFromContextOrDefault(ctx)
|
||||
}
|
||||
return ctx, r
|
||||
case *milvuspb.AlterCollectionSchemaRequest:
|
||||
if r.DbName == "" {
|
||||
r.DbName = GetCurDBNameFromContextOrDefault(ctx)
|
||||
|
||||
@@ -102,6 +102,7 @@ func TestDatabaseInterceptor(t *testing.T) {
|
||||
&milvuspb.SelectGrantRequest{Entity: &milvuspb.GrantEntity{}},
|
||||
&milvuspb.ManualCompactionRequest{},
|
||||
&milvuspb.AddCollectionFieldRequest{},
|
||||
&milvuspb.AddCollectionStructFieldRequest{},
|
||||
&milvuspb.AlterCollectionSchemaRequest{},
|
||||
&milvuspb.RunAnalyzerRequest{},
|
||||
&milvuspb.RefreshExternalCollectionRequest{},
|
||||
|
||||
@@ -1027,6 +1027,75 @@ func (node *Proxy) AddCollectionField(ctx context.Context, request *milvuspb.Add
|
||||
return task.result, nil
|
||||
}
|
||||
|
||||
// AddCollectionStructField add a struct field to collection
|
||||
func (node *Proxy) AddCollectionStructField(ctx context.Context, request *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, "Proxy-AddCollectionStructField")
|
||||
defer sp.End()
|
||||
|
||||
dresp, err := node.DescribeCollection(ctx, &milvuspb.DescribeCollectionRequest{DbName: request.DbName, CollectionName: request.CollectionName})
|
||||
if err := merr.CheckRPCCall(dresp, err); err != nil {
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
if typeutil.IsExternalCollection(dresp.GetSchema()) {
|
||||
return merr.Status(merr.WrapErrParameterInvalidMsg(
|
||||
"add struct field operation is not supported for external collection %s", request.GetCollectionName())), nil
|
||||
}
|
||||
|
||||
task := &addCollectionStructFieldTask{
|
||||
ctx: ctx,
|
||||
Condition: NewTaskCondition(ctx),
|
||||
AddCollectionStructFieldRequest: request,
|
||||
mixCoord: node.mixCoord,
|
||||
oldSchema: dresp.GetSchema(),
|
||||
}
|
||||
|
||||
method := "AddCollectionStructField"
|
||||
tr := timerecord.NewTimeRecorder(method)
|
||||
|
||||
log := log.Ctx(ctx).With(
|
||||
zap.String("role", typeutil.ProxyRole),
|
||||
zap.String("db", request.DbName),
|
||||
zap.String("collection", request.CollectionName))
|
||||
|
||||
log.Info(rpcReceived(method))
|
||||
|
||||
if err := node.sched.ddQueue.Enqueue(task); err != nil {
|
||||
log.Warn(
|
||||
rpcFailedToEnqueue(method),
|
||||
zap.Error(err))
|
||||
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
log.Info(
|
||||
rpcEnqueued(method),
|
||||
zap.Uint64("BeginTs", task.BeginTs()),
|
||||
zap.Uint64("EndTs", task.EndTs()))
|
||||
|
||||
if err := task.WaitToFinish(); err != nil {
|
||||
log.Warn(
|
||||
rpcFailedToWaitToFinish(method),
|
||||
zap.Error(err),
|
||||
zap.Uint64("BeginTs", task.BeginTs()),
|
||||
zap.Uint64("EndTs", task.EndTs()))
|
||||
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
log.Info(
|
||||
rpcDone(method),
|
||||
zap.Uint64("BeginTs", task.BeginTs()),
|
||||
zap.Uint64("EndTs", task.EndTs()))
|
||||
|
||||
metrics.ProxyReqLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method).Observe(float64(tr.ElapseSpan().Milliseconds()))
|
||||
return task.result, nil
|
||||
}
|
||||
|
||||
func (node *Proxy) AlterCollectionSchema(ctx context.Context, request *milvuspb.AlterCollectionSchemaRequest) (*milvuspb.AlterCollectionSchemaResponse, error) {
|
||||
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
|
||||
return &milvuspb.AlterCollectionSchemaResponse{
|
||||
|
||||
@@ -261,6 +261,10 @@ func (c *MockMixCoordClientInterface) AddCollectionField(ctx context.Context, in
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (c *MockMixCoordClientInterface) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
// describeCollectionInternal return collection info
|
||||
func (c *MockMixCoordClientInterface) describeCollectionInternal(ctx context.Context, in *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error) {
|
||||
panic("implement me")
|
||||
|
||||
@@ -394,6 +394,54 @@ func (coord *MixCoordMock) AddCollectionField(ctx context.Context, req *milvuspb
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
func (coord *MixCoordMock) AddCollectionStructField(ctx context.Context, req *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
code := coord.state.Load().(commonpb.StateCode)
|
||||
if code != commonpb.StateCode_Healthy {
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
||||
Reason: fmt.Sprintf("state code = %s", commonpb.StateCode_name[int32(code)]),
|
||||
}, nil
|
||||
}
|
||||
coord.collMtx.Lock()
|
||||
defer coord.collMtx.Unlock()
|
||||
|
||||
collID, exist := coord.collName2ID[req.CollectionName]
|
||||
if !exist {
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_CollectionNotExists,
|
||||
Reason: "collection not exist",
|
||||
}, nil
|
||||
}
|
||||
|
||||
collInfo, exist := coord.collID2Meta[collID]
|
||||
if !exist {
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_CollectionNotExists,
|
||||
Reason: "collection info not exist",
|
||||
}, nil
|
||||
}
|
||||
structField := proto.Clone(req.GetStructArrayFieldSchema()).(*schemapb.StructArrayFieldSchema)
|
||||
fieldIDStart := int64(common.StartOfUserFieldID + typeutil.GetTotalFieldsNum(collInfo.schema) + 1)
|
||||
structField.FieldID = fieldIDStart
|
||||
for i, field := range structField.GetFields() {
|
||||
field.FieldID = fieldIDStart + int64(i) + 1
|
||||
}
|
||||
collInfo.schema.StructArrayFields = append(collInfo.schema.StructArrayFields, structField)
|
||||
ts := uint64(time.Now().Nanosecond())
|
||||
coord.collID2Meta[collID] = collectionMeta{
|
||||
name: req.CollectionName,
|
||||
id: collID,
|
||||
schema: collInfo.schema,
|
||||
shardsNum: collInfo.shardsNum,
|
||||
virtualChannelNames: collInfo.virtualChannelNames,
|
||||
physicalChannelNames: collInfo.physicalChannelNames,
|
||||
createdTimestamp: ts,
|
||||
createdUtcTimestamp: ts,
|
||||
properties: collInfo.properties,
|
||||
}
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
func (coord *MixCoordMock) CreateCollection(ctx context.Context, req *milvuspb.CreateCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
code := coord.state.Load().(commonpb.StateCode)
|
||||
if code != commonpb.StateCode_Healthy {
|
||||
|
||||
+210
-3
@@ -133,6 +133,7 @@ const (
|
||||
DescribeDatabaseTaskName = "DescribeDatabaseTaskName"
|
||||
|
||||
AddFieldTaskName = "AddFieldTaskName"
|
||||
AddStructFieldTaskName = "AddStructFieldTaskName"
|
||||
AlterCollectionSchemaTaskName = "AlterCollectionSchemaTaskName"
|
||||
|
||||
// minFloat32 minimum float.
|
||||
@@ -700,15 +701,221 @@ func (t *addCollectionFieldTask) PostExecute(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type addCollectionStructFieldTask struct {
|
||||
baseTask
|
||||
Condition
|
||||
*milvuspb.AddCollectionStructFieldRequest
|
||||
ctx context.Context
|
||||
mixCoord types.MixCoordClient
|
||||
result *commonpb.Status
|
||||
structFieldSchema *schemapb.StructArrayFieldSchema
|
||||
oldSchema *schemapb.CollectionSchema
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) TraceCtx() context.Context {
|
||||
return t.ctx
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) ID() UniqueID {
|
||||
return t.Base.MsgID
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) SetID(uid UniqueID) {
|
||||
t.Base.MsgID = uid
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) Name() string {
|
||||
return AddStructFieldTaskName
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) Type() commonpb.MsgType {
|
||||
return t.Base.MsgType
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) BeginTs() Timestamp {
|
||||
return t.Base.Timestamp
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) EndTs() Timestamp {
|
||||
return t.Base.Timestamp
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) SetTs(ts Timestamp) {
|
||||
t.Base.Timestamp = ts
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) OnEnqueue() error {
|
||||
if t.Base == nil {
|
||||
t.Base = commonpbutil.NewMsgBase()
|
||||
}
|
||||
t.Base.MsgType = commonpb.MsgType_AddCollectionField
|
||||
t.Base.SourceID = paramtable.GetNodeID()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) PreExecute(ctx context.Context) error {
|
||||
if t.oldSchema == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("empty old schema in add struct field task")
|
||||
}
|
||||
if t.GetStructArrayFieldSchema() == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("struct array field schema is nil")
|
||||
}
|
||||
|
||||
t.structFieldSchema = proto.Clone(t.GetStructArrayFieldSchema()).(*schemapb.StructArrayFieldSchema)
|
||||
if err := validateAddStructFieldRequest(t.oldSchema, t.structFieldSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
transformStructArrayFieldSubNames(t.structFieldSchema)
|
||||
t.StructArrayFieldSchema = t.structFieldSchema
|
||||
|
||||
log.Info("PreExecute addStructField task done", zap.Any("struct field schema", t.structFieldSchema))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) Execute(ctx context.Context) error {
|
||||
var err error
|
||||
t.result, err = t.mixCoord.AddCollectionStructField(ctx, t.AddCollectionStructFieldRequest)
|
||||
return merr.CheckRPCCall(t.result, err)
|
||||
}
|
||||
|
||||
func (t *addCollectionStructFieldTask) PostExecute(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAddStructFieldRequest(schema *schemapb.CollectionSchema, structFieldSchema *schemapb.StructArrayFieldSchema) error {
|
||||
if schema == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("empty old schema in add struct field task")
|
||||
}
|
||||
if structFieldSchema == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("struct array field schema is nil")
|
||||
}
|
||||
if !structFieldSchema.GetNullable() {
|
||||
return merr.WrapErrParameterInvalidMsg("added struct field must be nullable, please check it, struct field name = %s", structFieldSchema.GetName())
|
||||
}
|
||||
if err := validateFieldName(structFieldSchema.GetName()); err != nil {
|
||||
return err
|
||||
}
|
||||
if funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, structFieldSchema.GetName()) {
|
||||
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", structFieldSchema.GetName())
|
||||
}
|
||||
if err := ValidateStructArrayField(structFieldSchema, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAddStructSubFieldProperties(structFieldSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAddStructFieldNames(schema, structFieldSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
totalFieldsNum := typeutil.GetTotalFieldsNum(schema) + len(structFieldSchema.GetFields()) + 1
|
||||
if totalFieldsNum > Params.ProxyCfg.MaxFieldNum.GetAsInt() {
|
||||
return fmt.Errorf("maximum field's number should be limited to %d", Params.ProxyCfg.MaxFieldNum.GetAsInt())
|
||||
}
|
||||
|
||||
vectorFields := len(typeutil.GetVectorFieldSchemas(schema))
|
||||
for _, subField := range structFieldSchema.GetFields() {
|
||||
if typeutil.IsVectorType(subField.GetDataType()) {
|
||||
vectorFields++
|
||||
}
|
||||
}
|
||||
if vectorFields > Params.ProxyCfg.MaxVectorFieldNum.GetAsInt() {
|
||||
return fmt.Errorf("maximum vector field's number should be limited to %d", Params.ProxyCfg.MaxVectorFieldNum.GetAsInt())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAddStructSubFieldProperties(structFieldSchema *schemapb.StructArrayFieldSchema) error {
|
||||
for _, subField := range structFieldSchema.GetFields() {
|
||||
if funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, subField.GetName()) {
|
||||
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", subField.GetName())
|
||||
}
|
||||
if subField.GetIsPrimaryKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("primary key is not supported for struct field, field name = %s", subField.GetName())
|
||||
}
|
||||
if subField.GetAutoID() {
|
||||
return merr.WrapErrParameterInvalidMsg("autoID is not supported for struct field, field name = %s", subField.GetName())
|
||||
}
|
||||
if subField.GetIsPartitionKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("partition key is not supported for struct field, field name = %s", subField.GetName())
|
||||
}
|
||||
if subField.GetIsClusteringKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("clustering key is not supported for struct field, field name = %s", subField.GetName())
|
||||
}
|
||||
if subField.GetDefaultValue() != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("default value is not supported for struct field, field name = %s", subField.GetName())
|
||||
}
|
||||
if subField.GetIsFunctionOutput() {
|
||||
return merr.WrapErrParameterInvalidMsg("function output is not supported for struct field, field name = %s", subField.GetName())
|
||||
}
|
||||
if subField.GetExternalField() != "" {
|
||||
return merr.WrapErrParameterInvalidMsg("add struct field operation does not support external field mapping, field name = %s", subField.GetName())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAddStructFieldNames(schema *schemapb.CollectionSchema, structFieldSchema *schemapb.StructArrayFieldSchema) error {
|
||||
existingNames := typeutil.NewSet[string]()
|
||||
for _, field := range schema.GetFields() {
|
||||
existingNames.Insert(field.GetName())
|
||||
}
|
||||
for _, structArrayField := range schema.GetStructArrayFields() {
|
||||
existingNames.Insert(structArrayField.GetName())
|
||||
for _, subField := range structArrayField.GetFields() {
|
||||
if typeutil.IsStructSubField(subField.GetName()) {
|
||||
existingNames.Insert(subField.GetName())
|
||||
}
|
||||
existingNames.Insert(storedStructSubFieldName(structArrayField.GetName(), subField.GetName()))
|
||||
}
|
||||
}
|
||||
|
||||
if existingNames.Contain(structFieldSchema.GetName()) {
|
||||
return merr.WrapErrParameterInvalidMsg("duplicated field name %s", structFieldSchema.GetName())
|
||||
}
|
||||
|
||||
newSubFieldNames := make(map[string]struct{})
|
||||
for _, subField := range structFieldSchema.GetFields() {
|
||||
if _, ok := newSubFieldNames[subField.GetName()]; ok {
|
||||
return merr.WrapErrParameterInvalidMsg("duplicated field name %s", subField.GetName())
|
||||
}
|
||||
newSubFieldNames[subField.GetName()] = struct{}{}
|
||||
|
||||
transformedName := typeutil.ConcatStructFieldName(structFieldSchema.GetName(), subField.GetName())
|
||||
if existingNames.Contain(transformedName) {
|
||||
return merr.WrapErrParameterInvalidMsg("duplicated field name %s", transformedName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func storedStructSubFieldName(structName string, fieldName string) string {
|
||||
if typeutil.IsStructSubField(fieldName) {
|
||||
return fieldName
|
||||
}
|
||||
return typeutil.ConcatStructFieldName(structName, fieldName)
|
||||
}
|
||||
|
||||
func transformStructArrayFieldSubNames(structFieldSchema *schemapb.StructArrayFieldSchema) {
|
||||
structName := structFieldSchema.GetName()
|
||||
for _, field := range structFieldSchema.GetFields() {
|
||||
field.Name = typeutil.ConcatStructFieldName(structName, field.GetName())
|
||||
}
|
||||
}
|
||||
|
||||
// validateAddFieldRequest validates both the old schema constraints and the new field properties
|
||||
// for an AddCollectionField request. It is the single source of truth for add-field validation.
|
||||
func validateAddFieldRequest(schema *schemapb.CollectionSchema, newFieldSchema *schemapb.FieldSchema) error {
|
||||
// --- old schema constraints ---
|
||||
fieldList := typeutil.NewSet[string]()
|
||||
for _, f := range schema.Fields {
|
||||
fieldList.Insert(f.Name)
|
||||
for _, field := range schema.GetFields() {
|
||||
fieldList.Insert(field.GetName())
|
||||
}
|
||||
if len(fieldList) >= Params.ProxyCfg.MaxFieldNum.GetAsInt() {
|
||||
for _, structArrayField := range schema.GetStructArrayFields() {
|
||||
fieldList.Insert(structArrayField.GetName())
|
||||
}
|
||||
if typeutil.GetTotalFieldsNum(schema) >= Params.ProxyCfg.MaxFieldNum.GetAsInt() {
|
||||
msg := fmt.Sprintf("The number of fields has reached the maximum value %d", Params.ProxyCfg.MaxFieldNum.GetAsInt())
|
||||
return merr.WrapErrParameterInvalidMsg(msg)
|
||||
}
|
||||
|
||||
@@ -1257,6 +1257,142 @@ func TestAddFieldTask(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestAddCollectionStructFieldTaskPreExecute(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
oldSchema := &schemapb.CollectionSchema{
|
||||
Name: "test_collection",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{
|
||||
FieldID: 100,
|
||||
Name: "pk",
|
||||
DataType: schemapb.DataType_Int64,
|
||||
IsPrimaryKey: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
structField := newAddStructFieldSchema("profile")
|
||||
|
||||
task := &addCollectionStructFieldTask{
|
||||
Condition: NewTaskCondition(ctx),
|
||||
AddCollectionStructFieldRequest: &milvuspb.AddCollectionStructFieldRequest{
|
||||
DbName: "",
|
||||
CollectionName: oldSchema.GetName(),
|
||||
StructArrayFieldSchema: structField,
|
||||
},
|
||||
ctx: ctx,
|
||||
mixCoord: NewMixCoordMock(),
|
||||
oldSchema: oldSchema,
|
||||
}
|
||||
|
||||
require.NoError(t, task.OnEnqueue())
|
||||
assert.Equal(t, commonpb.MsgType_AddCollectionField, task.Type())
|
||||
|
||||
require.NoError(t, task.PreExecute(ctx))
|
||||
|
||||
got := task.GetStructArrayFieldSchema()
|
||||
require.NotSame(t, structField, got)
|
||||
require.Len(t, got.GetFields(), 2)
|
||||
assert.Equal(t, "profile[ints]", got.GetFields()[0].GetName())
|
||||
assert.Equal(t, "profile[vectors]", got.GetFields()[1].GetName())
|
||||
assert.True(t, got.GetFields()[0].GetNullable())
|
||||
assert.True(t, got.GetFields()[1].GetNullable())
|
||||
|
||||
// PreExecute works on a cloned schema, so callers do not observe partial mutation on failure.
|
||||
assert.Equal(t, "ints", structField.GetFields()[0].GetName())
|
||||
assert.Equal(t, "vectors", structField.GetFields()[1].GetName())
|
||||
}
|
||||
|
||||
func TestValidateAddStructFieldRequest(t *testing.T) {
|
||||
baseSchema := &schemapb.CollectionSchema{
|
||||
Name: "test_collection",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{
|
||||
Name: "pk",
|
||||
DataType: schemapb.DataType_Int64,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("profile"))
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("struct must be nullable", func(t *testing.T) {
|
||||
structField := newAddStructFieldSchema("profile")
|
||||
structField.Nullable = false
|
||||
|
||||
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), structField)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
|
||||
assert.Contains(t, err.Error(), "added struct field must be nullable")
|
||||
})
|
||||
|
||||
t.Run("duplicate parent name", func(t *testing.T) {
|
||||
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("pk"))
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
|
||||
assert.Contains(t, err.Error(), "duplicated field name pk")
|
||||
})
|
||||
|
||||
t.Run("duplicate transformed sub field name", func(t *testing.T) {
|
||||
schema := proto.Clone(baseSchema).(*schemapb.CollectionSchema)
|
||||
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{
|
||||
Name: "profile[ints]",
|
||||
DataType: schemapb.DataType_Int64,
|
||||
})
|
||||
|
||||
err := validateAddStructFieldRequest(schema, newAddStructFieldSchema("profile"))
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
|
||||
assert.Contains(t, err.Error(), "duplicated field name profile[ints]")
|
||||
})
|
||||
|
||||
t.Run("max field count includes struct parent", func(t *testing.T) {
|
||||
Params.Save(Params.ProxyCfg.MaxFieldNum.Key, "3")
|
||||
defer Params.Reset(Params.ProxyCfg.MaxFieldNum.Key)
|
||||
|
||||
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("profile"))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "maximum field's number should be limited to 3")
|
||||
})
|
||||
|
||||
t.Run("vector count includes array of vector sub field", func(t *testing.T) {
|
||||
Params.Save(Params.ProxyCfg.MaxVectorFieldNum.Key, "0")
|
||||
defer Params.Reset(Params.ProxyCfg.MaxVectorFieldNum.Key)
|
||||
|
||||
err := validateAddStructFieldRequest(proto.Clone(baseSchema).(*schemapb.CollectionSchema), newAddStructFieldSchema("profile"))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "maximum vector field's number should be limited to 0")
|
||||
})
|
||||
}
|
||||
|
||||
func newAddStructFieldSchema(name string) *schemapb.StructArrayFieldSchema {
|
||||
return &schemapb.StructArrayFieldSchema{
|
||||
Name: name,
|
||||
Nullable: true,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{
|
||||
Name: "ints",
|
||||
DataType: schemapb.DataType_Array,
|
||||
ElementType: schemapb.DataType_Int64,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxCapacityKey, Value: "16"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "vectors",
|
||||
DataType: schemapb.DataType_ArrayOfVector,
|
||||
ElementType: schemapb.DataType_FloatVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "8"},
|
||||
{Key: common.MaxCapacityKey, Value: "16"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCollectionTask(t *testing.T) {
|
||||
mix := NewMixCoordMock()
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -28,6 +28,7 @@ const (
|
||||
globalIDAllocatorSubPath = "gid"
|
||||
globalTSOAllocatorKey = "timestamp"
|
||||
globalTSOAllocatorSubPath = "tso"
|
||||
defaultMaxArrayCapacity = 4096
|
||||
)
|
||||
|
||||
func checkGeneralCapacity(ctx context.Context, newColNum int,
|
||||
|
||||
@@ -52,12 +52,21 @@ func (c *Core) broadcastAlterCollectionForAddField(ctx context.Context, req *mil
|
||||
}
|
||||
|
||||
// check if the field already exists
|
||||
fieldNames := typeutil.NewSet[string]()
|
||||
for _, field := range coll.Fields {
|
||||
if field.Name == fieldSchema.Name {
|
||||
// TODO: idempotency check here.
|
||||
return merr.WrapErrParameterInvalidMsg("field already exists, name: %s", fieldSchema.Name)
|
||||
fieldNames.Insert(field.Name)
|
||||
}
|
||||
for _, structField := range coll.StructArrayFields {
|
||||
fieldNames.Insert(structField.Name)
|
||||
for _, field := range structField.Fields {
|
||||
fieldNames.Insert(field.Name)
|
||||
fieldNames.Insert(storedRootStructSubFieldName(structField.Name, field.Name))
|
||||
}
|
||||
}
|
||||
if fieldNames.Contain(fieldSchema.Name) {
|
||||
// TODO: idempotency check here.
|
||||
return merr.WrapErrParameterInvalidMsg("field already exists, name: %s", fieldSchema.Name)
|
||||
}
|
||||
|
||||
// assign a new field id.
|
||||
fieldSchema.FieldID = nextFieldID(coll)
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rootcoord
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/fieldmaskpb"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/distributed/streaming"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
// broadcastAlterCollectionForAddStructField broadcasts the put collection message for add struct field.
|
||||
func (c *Core) broadcastAlterCollectionForAddStructField(ctx context.Context, req *milvuspb.AddCollectionStructFieldRequest) error {
|
||||
broadcaster, err := c.startBroadcastWithAliasOrCollectionLock(ctx, req.GetDbName(), req.GetCollectionName())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer broadcaster.Close()
|
||||
|
||||
coll, err := c.meta.GetCollectionByName(ctx, req.GetDbName(), req.GetCollectionName(), typeutil.MaxTimestamp, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if req.GetStructArrayFieldSchema() == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("struct array field schema is nil")
|
||||
}
|
||||
structArrayField := proto.Clone(req.GetStructArrayFieldSchema()).(*schemapb.StructArrayFieldSchema)
|
||||
if err := normalizeAndCheckAddedStructField(structArrayField); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkStructArrayFieldSchema([]*schemapb.StructArrayFieldSchema{structArrayField}); err != nil {
|
||||
return errors.Wrap(err, "failed to check struct array field schema")
|
||||
}
|
||||
if err := validateStructArrayFieldDataType([]*schemapb.StructArrayFieldSchema{structArrayField}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkAddStructFieldDuplicate(coll, structArrayField); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fieldIDStart := nextFieldID(coll)
|
||||
structArrayField.FieldID = fieldIDStart
|
||||
for i, field := range structArrayField.GetFields() {
|
||||
field.FieldID = fieldIDStart + int64(i) + 1
|
||||
}
|
||||
|
||||
schema := coll.ToCollectionSchemaPB()
|
||||
schema.Version = coll.SchemaVersion + 1
|
||||
schema.StructArrayFields = append(schema.StructArrayFields, structArrayField)
|
||||
|
||||
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
channels := make([]string, 0, len(coll.VirtualChannelNames)+1)
|
||||
channels = append(channels, streaming.WAL().ControlChannel())
|
||||
channels = append(channels, coll.VirtualChannelNames...)
|
||||
msg := message.NewAlterCollectionMessageBuilderV2().
|
||||
WithHeader(&messagespb.AlterCollectionMessageHeader{
|
||||
DbId: coll.DBID,
|
||||
CollectionId: coll.CollectionID,
|
||||
UpdateMask: &fieldmaskpb.FieldMask{
|
||||
Paths: []string{message.FieldMaskCollectionSchema},
|
||||
},
|
||||
CacheExpirations: cacheExpirations,
|
||||
}).
|
||||
WithBody(&messagespb.AlterCollectionMessageBody{
|
||||
Updates: &messagespb.AlterCollectionMessageUpdates{
|
||||
Schema: schema,
|
||||
},
|
||||
}).
|
||||
WithBroadcast(channels).
|
||||
MustBuildBroadcast()
|
||||
if _, err := broadcaster.Broadcast(ctx, msg); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeAndCheckAddedStructField(structArrayField *schemapb.StructArrayFieldSchema) error {
|
||||
if !structArrayField.GetNullable() {
|
||||
return merr.WrapErrParameterInvalidMsg("added struct field must be nullable, please check it, struct field name = %s", structArrayField.GetName())
|
||||
}
|
||||
if err := validateAddedStructFieldName(structArrayField.GetName()); err != nil {
|
||||
return err
|
||||
}
|
||||
if isReservedStructFieldName(structArrayField.GetName()) {
|
||||
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", structArrayField.GetName())
|
||||
}
|
||||
|
||||
for _, field := range structArrayField.GetFields() {
|
||||
originalName, err := normalizeStructSubFieldName(structArrayField.GetName(), field)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAddedStructFieldName(originalName); err != nil {
|
||||
return err
|
||||
}
|
||||
if isReservedStructFieldName(originalName) {
|
||||
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", originalName)
|
||||
}
|
||||
if field.GetIsPrimaryKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("primary key is not supported for struct field, field name = %s", originalName)
|
||||
}
|
||||
if field.GetAutoID() {
|
||||
return merr.WrapErrParameterInvalidMsg("autoID is not supported for struct field, field name = %s", originalName)
|
||||
}
|
||||
if field.GetIsPartitionKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("partition key is not supported for struct field, field name = %s", originalName)
|
||||
}
|
||||
if field.GetIsClusteringKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("clustering key is not supported for struct field, field name = %s", originalName)
|
||||
}
|
||||
if field.GetDefaultValue() != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("default value is not supported for struct field, field name = %s", originalName)
|
||||
}
|
||||
if field.GetIsFunctionOutput() {
|
||||
return merr.WrapErrParameterInvalidMsg("function output is not supported for struct field, field name = %s", originalName)
|
||||
}
|
||||
if field.GetExternalField() != "" {
|
||||
return merr.WrapErrParameterInvalidMsg("add struct field operation does not support external field mapping, field name = %s", originalName)
|
||||
}
|
||||
field.Nullable = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeStructSubFieldName(structName string, field *schemapb.FieldSchema) (string, error) {
|
||||
fieldName := field.GetName()
|
||||
if isStoredStructSubFieldName(structName, fieldName) {
|
||||
originalName, err := typeutil.ExtractStructFieldName(fieldName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return originalName, nil
|
||||
}
|
||||
if typeutil.IsStructSubField(fieldName) {
|
||||
return "", merr.WrapErrParameterInvalidMsg("invalid struct sub-field name %s for struct field %s", fieldName, structName)
|
||||
}
|
||||
|
||||
field.Name = typeutil.ConcatStructFieldName(structName, fieldName)
|
||||
return fieldName, nil
|
||||
}
|
||||
|
||||
func checkAddStructFieldDuplicate(coll *model.Collection, structArrayField *schemapb.StructArrayFieldSchema) error {
|
||||
fieldNames := make(map[string]struct{})
|
||||
addFieldName := func(name string) error {
|
||||
if _, ok := fieldNames[name]; ok {
|
||||
return merr.WrapErrParameterInvalidMsg("field already exists, name: %s", name)
|
||||
}
|
||||
fieldNames[name] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, field := range coll.Fields {
|
||||
if err := addFieldName(field.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, structField := range coll.StructArrayFields {
|
||||
if err := addFieldName(structField.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, field := range structField.Fields {
|
||||
if err := addFieldName(field.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
storedName := storedRootStructSubFieldName(structField.Name, field.Name)
|
||||
if storedName != field.Name {
|
||||
if err := addFieldName(storedName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := addFieldName(structArrayField.GetName()); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, field := range structArrayField.GetFields() {
|
||||
if err := addFieldName(field.GetName()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isStoredStructSubFieldName(structName string, fieldName string) bool {
|
||||
return strings.HasPrefix(fieldName, structName+"[") && strings.HasSuffix(fieldName, "]")
|
||||
}
|
||||
|
||||
func storedRootStructSubFieldName(structName string, fieldName string) string {
|
||||
if isStoredStructSubFieldName(structName, fieldName) {
|
||||
return fieldName
|
||||
}
|
||||
return typeutil.ConcatStructFieldName(structName, fieldName)
|
||||
}
|
||||
|
||||
func isReservedStructFieldName(fieldName string) bool {
|
||||
return funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, fieldName)
|
||||
}
|
||||
|
||||
func validateAddedStructFieldName(fieldName string) error {
|
||||
fieldName = strings.TrimSpace(fieldName)
|
||||
if fieldName == "" {
|
||||
return merr.WrapErrFieldNameInvalid(fieldName, "field name should not be empty")
|
||||
}
|
||||
|
||||
invalidMsg := "Invalid field name: " + fieldName + ". "
|
||||
if len(fieldName) > Params.ProxyCfg.MaxNameLength.GetAsInt() {
|
||||
msg := invalidMsg + "The length of a field name must be less than " + Params.ProxyCfg.MaxNameLength.GetValue() + " characters."
|
||||
return merr.WrapErrFieldNameInvalid(fieldName, msg)
|
||||
}
|
||||
|
||||
firstChar := fieldName[0]
|
||||
if firstChar != '_' && !isAlphaForStructFieldName(firstChar) {
|
||||
msg := invalidMsg + "The first character of a field name must be an underscore or letter."
|
||||
return merr.WrapErrFieldNameInvalid(fieldName, msg)
|
||||
}
|
||||
|
||||
for i := 1; i < len(fieldName); i++ {
|
||||
c := fieldName[i]
|
||||
if c != '_' && !isAlphaForStructFieldName(c) && !isNumberForStructFieldName(c) {
|
||||
msg := invalidMsg + "Field name can only contain numbers, letters, and underscores."
|
||||
return merr.WrapErrFieldNameInvalid(fieldName, msg)
|
||||
}
|
||||
}
|
||||
if _, ok := common.FieldNameKeywords[fieldName]; ok {
|
||||
msg := invalidMsg + fmt.Sprintf("%s is keyword in milvus.", fieldName)
|
||||
return merr.WrapErrFieldNameInvalid(fieldName, msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isAlphaForStructFieldName(c uint8) bool {
|
||||
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
|
||||
}
|
||||
|
||||
func isNumberForStructFieldName(c uint8) bool {
|
||||
return c >= '0' && c <= '9'
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rootcoord
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
func TestDDLCallbacksAlterCollectionAddStructField(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
|
||||
ctx := context.Background()
|
||||
dbName := "testDB" + funcutil.RandomString(10)
|
||||
collectionName := "testCollection" + funcutil.RandomString(10)
|
||||
createCollectionForTest(t, ctx, core, dbName, collectionName)
|
||||
|
||||
resp, err := core.AddCollectionStructField(ctx, &milvuspb.AddCollectionStructFieldRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
StructArrayFieldSchema: newRootAddStructFieldSchema("field1"),
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
|
||||
|
||||
resp, err = core.AddCollectionStructField(ctx, &milvuspb.AddCollectionStructFieldRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
StructArrayFieldSchema: newRootAddStructFieldSchema("profile"),
|
||||
})
|
||||
require.NoError(t, merr.CheckRPCCall(resp, err))
|
||||
|
||||
coll, err := core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, getUserFields(coll.Fields), 1)
|
||||
require.Len(t, coll.StructArrayFields, 1)
|
||||
|
||||
structField := coll.StructArrayFields[0]
|
||||
require.Equal(t, int64(101), structField.FieldID)
|
||||
require.Equal(t, "profile", structField.Name)
|
||||
require.True(t, structField.Nullable)
|
||||
require.Len(t, structField.Fields, 2)
|
||||
|
||||
require.Equal(t, int64(102), structField.Fields[0].FieldID)
|
||||
require.Equal(t, "profile[ints]", structField.Fields[0].Name)
|
||||
require.Equal(t, schemapb.DataType_Array, structField.Fields[0].DataType)
|
||||
require.True(t, structField.Fields[0].Nullable)
|
||||
|
||||
require.Equal(t, int64(103), structField.Fields[1].FieldID)
|
||||
require.Equal(t, "profile[vectors]", structField.Fields[1].Name)
|
||||
require.Equal(t, schemapb.DataType_ArrayOfVector, structField.Fields[1].DataType)
|
||||
require.True(t, structField.Fields[1].Nullable)
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 1)
|
||||
|
||||
resp, err = core.AddCollectionStructField(ctx, &milvuspb.AddCollectionStructFieldRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
StructArrayFieldSchema: newRootAddStructFieldSchema("profile"),
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
|
||||
|
||||
resp, err = core.AddCollectionField(ctx, &milvuspb.AddCollectionFieldRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
Schema: getFieldSchema("profile"),
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
|
||||
}
|
||||
|
||||
func getUserFields(fields []*model.Field) []*model.Field {
|
||||
ret := make([]*model.Field, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
if field.Name == RowIDFieldName || field.Name == TimeStampFieldName {
|
||||
continue
|
||||
}
|
||||
ret = append(ret, field)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func newRootAddStructFieldSchema(name string) *schemapb.StructArrayFieldSchema {
|
||||
return &schemapb.StructArrayFieldSchema{
|
||||
Name: name,
|
||||
Nullable: true,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{
|
||||
Name: typeutil.ConcatStructFieldName(name, "ints"),
|
||||
DataType: schemapb.DataType_Array,
|
||||
ElementType: schemapb.DataType_Int64,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxCapacityKey, Value: "16"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: typeutil.ConcatStructFieldName(name, "vectors"),
|
||||
DataType: schemapb.DataType_ArrayOfVector,
|
||||
ElementType: schemapb.DataType_FloatVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "8"},
|
||||
{Key: common.MaxCapacityKey, Value: "16"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1025,6 +1025,32 @@ func (c *Core) AddCollectionField(ctx context.Context, in *milvuspb.AddCollectio
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
// AddCollectionStructField add struct field
|
||||
func (c *Core) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
if err := merr.CheckHealthy(c.GetStateCode()); err != nil {
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionStructField", metrics.TotalLabel).Inc()
|
||||
tr := timerecord.NewTimeRecorder("AddCollectionStructField")
|
||||
|
||||
log := log.Ctx(ctx).With(zap.String("role", typeutil.RootCoordRole),
|
||||
zap.String("dbName", in.GetDbName()),
|
||||
zap.String("collectionName", in.GetCollectionName()))
|
||||
log.Info("received request to add collection struct field")
|
||||
|
||||
if err := c.broadcastAlterCollectionForAddStructField(ctx, in); err != nil {
|
||||
log.Info("failed to add collection struct field", zap.Error(err))
|
||||
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionStructField", metrics.FailLabel).Inc()
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionStructField", metrics.SuccessLabel).Inc()
|
||||
metrics.RootCoordDDLReqLatency.WithLabelValues("AddCollectionStructField").Observe(float64(tr.ElapseSpan().Milliseconds()))
|
||||
log.Info("done to add collection struct field")
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
func (c *Core) AlterCollectionSchema(ctx context.Context, in *milvuspb.AlterCollectionSchemaRequest) (*milvuspb.AlterCollectionSchemaResponse, error) {
|
||||
if err := merr.CheckHealthy(c.GetStateCode()); err != nil {
|
||||
return &milvuspb.AlterCollectionSchemaResponse{
|
||||
|
||||
@@ -507,6 +507,9 @@ func getStructSubFieldMaxCapacity(structName string, field *schemapb.FieldSchema
|
||||
return 0, merr.WrapErrParameterInvalidMsg("the value for %s of field %s in struct array field %s must be an integer",
|
||||
common.MaxCapacityKey, field.GetName(), structName)
|
||||
}
|
||||
if maxCapacity > defaultMaxArrayCapacity || maxCapacity <= 0 {
|
||||
return 0, merr.WrapErrParameterInvalidMsg("the maximum capacity specified for a Array should be in (0, %d]", defaultMaxArrayCapacity)
|
||||
}
|
||||
return maxCapacity, nil
|
||||
}
|
||||
return 0, merr.WrapErrParameterInvalidMsg("type param(%s) should be specified for field %s in struct array field %s",
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
func ConvertToArrowSchema(schema *schemapb.CollectionSchema, useFieldID bool) (*arrow.Schema, error) {
|
||||
fieldCount := typeutil.GetTotalFieldsNum(schema)
|
||||
fieldCount := len(typeutil.GetAllFieldSchemas(schema))
|
||||
arrowFields := make([]arrow.Field, 0, fieldCount)
|
||||
appendArrowField := func(field *schemapb.FieldSchema) error {
|
||||
if serdeMap[field.DataType].arrowType == nil {
|
||||
|
||||
@@ -93,6 +93,10 @@ func (m *GrpcRootCoordClient) AddCollectionField(ctx context.Context, in *milvus
|
||||
return &commonpb.Status{}, m.Err
|
||||
}
|
||||
|
||||
func (m *GrpcRootCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
return &commonpb.Status{}, m.Err
|
||||
}
|
||||
|
||||
func (m *GrpcRootCoordClient) ListPolicy(ctx context.Context, in *internalpb.ListPolicyRequest, opts ...grpc.CallOption) (*internalpb.ListPolicyResponse, error) {
|
||||
return &internalpb.ListPolicyResponse{}, m.Err
|
||||
}
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ require (
|
||||
github.com/jolestar/go-commons-pool/v2 v2.1.2
|
||||
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12
|
||||
github.com/klauspost/compress v1.18.0
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
|
||||
github.com/minio/minio-go/v7 v7.0.73
|
||||
github.com/panjf2000/ants/v2 v2.11.3
|
||||
github.com/prometheus/client_golang v1.20.5
|
||||
|
||||
+2
-2
@@ -517,8 +517,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6L
|
||||
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f h1:Tmo+3s4VHQ8EUGmuLoo5PKo7k3eF90w6hCk/acXG0oQ=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260428092345-c9f705f3cc7f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo=
|
||||
|
||||
@@ -49,6 +49,15 @@ service RootCoord {
|
||||
*/
|
||||
rpc AddCollectionField(milvus.AddCollectionFieldRequest) returns (common.Status) {}
|
||||
|
||||
/**
|
||||
* @brief This method is used to add collection struct field.
|
||||
*
|
||||
* @param AddCollectionStructFieldRequest, struct field schema is going to be added.
|
||||
*
|
||||
* @return Status
|
||||
*/
|
||||
rpc AddCollectionStructField(milvus.AddCollectionStructFieldRequest) returns (common.Status) {}
|
||||
|
||||
|
||||
/**
|
||||
* @brief This method is used to add/drop collection field/function.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,6 +30,7 @@ const (
|
||||
RootCoord_DropCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/DropCollection"
|
||||
RootCoord_TruncateCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/TruncateCollection"
|
||||
RootCoord_AddCollectionField_FullMethodName = "/milvus.proto.rootcoord.RootCoord/AddCollectionField"
|
||||
RootCoord_AddCollectionStructField_FullMethodName = "/milvus.proto.rootcoord.RootCoord/AddCollectionStructField"
|
||||
RootCoord_AlterCollectionSchema_FullMethodName = "/milvus.proto.rootcoord.RootCoord/AlterCollectionSchema"
|
||||
RootCoord_HasCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/HasCollection"
|
||||
RootCoord_DescribeCollection_FullMethodName = "/milvus.proto.rootcoord.RootCoord/DescribeCollection"
|
||||
@@ -133,6 +134,13 @@ type RootCoordClient interface {
|
||||
// @return Status
|
||||
AddCollectionField(ctx context.Context, in *milvuspb.AddCollectionFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
// *
|
||||
// @brief This method is used to add collection struct field.
|
||||
//
|
||||
// @param AddCollectionStructFieldRequest, struct field schema is going to be added.
|
||||
//
|
||||
// @return Status
|
||||
AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
// *
|
||||
// @brief This method is used to add/drop collection field/function.
|
||||
//
|
||||
// @param AlterCollectionSchemaRequest, field schema is going to be added.
|
||||
@@ -316,6 +324,15 @@ func (c *rootCoordClient) AddCollectionField(ctx context.Context, in *milvuspb.A
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *rootCoordClient) AddCollectionStructField(ctx context.Context, in *milvuspb.AddCollectionStructFieldRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
out := new(commonpb.Status)
|
||||
err := c.cc.Invoke(ctx, RootCoord_AddCollectionStructField_FullMethodName, in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *rootCoordClient) AlterCollectionSchema(ctx context.Context, in *milvuspb.AlterCollectionSchemaRequest, opts ...grpc.CallOption) (*milvuspb.AlterCollectionSchemaResponse, error) {
|
||||
out := new(milvuspb.AlterCollectionSchemaResponse)
|
||||
err := c.cc.Invoke(ctx, RootCoord_AlterCollectionSchema_FullMethodName, in, out, opts...)
|
||||
@@ -937,6 +954,13 @@ type RootCoordServer interface {
|
||||
// @return Status
|
||||
AddCollectionField(context.Context, *milvuspb.AddCollectionFieldRequest) (*commonpb.Status, error)
|
||||
// *
|
||||
// @brief This method is used to add collection struct field.
|
||||
//
|
||||
// @param AddCollectionStructFieldRequest, struct field schema is going to be added.
|
||||
//
|
||||
// @return Status
|
||||
AddCollectionStructField(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error)
|
||||
// *
|
||||
// @brief This method is used to add/drop collection field/function.
|
||||
//
|
||||
// @param AlterCollectionSchemaRequest, field schema is going to be added.
|
||||
@@ -1074,6 +1098,9 @@ func (UnimplementedRootCoordServer) TruncateCollection(context.Context, *milvusp
|
||||
func (UnimplementedRootCoordServer) AddCollectionField(context.Context, *milvuspb.AddCollectionFieldRequest) (*commonpb.Status, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method AddCollectionField not implemented")
|
||||
}
|
||||
func (UnimplementedRootCoordServer) AddCollectionStructField(context.Context, *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method AddCollectionStructField not implemented")
|
||||
}
|
||||
func (UnimplementedRootCoordServer) AlterCollectionSchema(context.Context, *milvuspb.AlterCollectionSchemaRequest) (*milvuspb.AlterCollectionSchemaResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method AlterCollectionSchema not implemented")
|
||||
}
|
||||
@@ -1407,6 +1434,24 @@ func _RootCoord_AddCollectionField_Handler(srv interface{}, ctx context.Context,
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _RootCoord_AddCollectionStructField_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(milvuspb.AddCollectionStructFieldRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(RootCoordServer).AddCollectionStructField(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: RootCoord_AddCollectionStructField_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(RootCoordServer).AddCollectionStructField(ctx, req.(*milvuspb.AddCollectionStructFieldRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _RootCoord_AlterCollectionSchema_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(milvuspb.AlterCollectionSchemaRequest)
|
||||
if err := dec(in); err != nil {
|
||||
@@ -2612,6 +2657,10 @@ var RootCoord_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "AddCollectionField",
|
||||
Handler: _RootCoord_AddCollectionField_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "AddCollectionStructField",
|
||||
Handler: _RootCoord_AddCollectionStructField_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "AlterCollectionSchema",
|
||||
Handler: _RootCoord_AlterCollectionSchema_Handler,
|
||||
|
||||
@@ -2197,12 +2197,12 @@ func MergeFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTotalFieldsNum get total fields number
|
||||
// We exclude StructArrayField itself as it does not contain data directly.
|
||||
// GetTotalFieldsNum get total fields number, including StructArrayField itself.
|
||||
func GetTotalFieldsNum(schema *schemapb.CollectionSchema) int {
|
||||
num := len(schema.GetFields())
|
||||
for _, structArrayField := range schema.GetStructArrayFields() {
|
||||
num += len(structArrayField.GetFields())
|
||||
// +1 for the StructArrayField itself
|
||||
num += len(structArrayField.GetFields()) + 1
|
||||
}
|
||||
return num
|
||||
}
|
||||
|
||||
@@ -5974,3 +5974,24 @@ func TestAppendFieldData_ArrayOfVectorNullRowAppendsPlaceholder(t *testing.T) {
|
||||
assert.EqualValues(t, 1, got.GetDim())
|
||||
assert.Equal(t, schemapb.DataType_FloatVector, got.GetElementType())
|
||||
}
|
||||
|
||||
func TestGetTotalFieldsNumIncludesStructParent(t *testing.T) {
|
||||
schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "pk", DataType: schemapb.DataType_Int64},
|
||||
{Name: "scalar", DataType: schemapb.DataType_Bool},
|
||||
},
|
||||
StructArrayFields: []*schemapb.StructArrayFieldSchema{
|
||||
{
|
||||
Name: "profile",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "profile[ints]", DataType: schemapb.DataType_Array},
|
||||
{Name: "profile[vectors]", DataType: schemapb.DataType_ArrayOfVector},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, 5, GetTotalFieldsNum(schema))
|
||||
assert.Len(t, GetAllFieldSchemas(schema), 4)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ go 1.25.9
|
||||
|
||||
require (
|
||||
github.com/apache/arrow/go/v17 v17.0.0
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f
|
||||
github.com/milvus-io/milvus/client/v2 v2.0.0-20241125024034-0b9edb62a92d
|
||||
github.com/milvus-io/milvus/pkg/v3 v3.0.0-beta
|
||||
github.com/minio/minio-go/v7 v7.0.73
|
||||
|
||||
@@ -245,8 +245,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710 h1:vEaS+KqeIJZbByJFLjVZVA+630oqD40u4jMtdj5FjyA=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260506064405-f5b77584c710/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f h1:Z2paGeFL0i1a3Dqw82DwRwZ2ix3FtyKU7+u6rR7jbUs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v3 v3.0.0-20260514121806-9b60a053353f/go.mod h1:rbKpv5JToISTKTTLl0duL5r6wbYnjJ9SsD0QgXMzKy0=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
||||
|
||||
Reference in New Issue
Block a user