enhance: add dynamic StructArray field APIs (#50276)

issue: #50235

### What problem does this PR solve?

This PR adds API support for dynamically adding StructArray fields to an
existing collection, aligned with the existing Python SDK API shape.

### What is changed?

- Add Go SDK `Client.AddCollectionStructField(...)` and
`NewAddCollectionStructFieldOption(...)`.
- Convert Go SDK StructArray field schema into
`AddCollectionStructFieldRequest`.
- Preserve StructArray parent `nullable` and `max_capacity` when
converting schemas and describing collections.
- Add REST v2 endpoint:
  - `POST /v2/vectordb/collections/struct_fields/add`
- Reject StructArray payloads from the ordinary REST v2 field endpoint:
  - `POST /v2/vectordb/collections/fields/add`
- Support REST v2 StructArray add payloads using both:
  - `dataType: "Array", elementDataType: "Struct"`
  - `dataType: "ArrayOfStruct"`
- Add Go SDK and REST v2 regression coverage.

### Verification

- `milvus-dev-cli go-ut local . -b master -t
internal/distributed/proxy/httpserver -k 'TestAddCollectionFieldSuite'
-f`
  - `ut-go-local-zhuwenxi-issue-50235-cbec6c2-26ef323b`
- `milvus-dev-cli build local . -b master -f`
  - `build-local-zhuwenxing-issue-50235-clea-cbec6c2`
- `milvus-dev-cli deploy create
build-local-zhuwenxing-issue-50235-clea-cbec6c2 --name
issue-50235-struct-add`
- `pytest -q
testcases/test_collection_operations.py::TestCollectionAddField::test_add_struct_array_field
--endpoint http://10.100.36.203:19530 --token root:Milvus --minio_host
10.100.36.198`
  - `2 passed`

---------

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
zhuwenxing
2026-06-12 15:30:21 +08:00
committed by GitHub
parent 9e2bfdabbc
commit 14c4cf65fc
14 changed files with 884 additions and 279 deletions
+9 -1
View File
@@ -154,6 +154,7 @@ func (s *Schema) ProtoMessage() *schemapb.CollectionSchema {
Name: field.Name,
Description: field.Description,
TypeParams: MapKvPairs(field.TypeParams),
Nullable: field.Nullable,
}
// max_capacity declared on the parent struct field must be carried onto each sub-field's
// type params — the server validates it per sub-field on insert.
@@ -222,6 +223,7 @@ func (s *Schema) ReadProto(p *schemapb.CollectionSchema) *Schema {
// original (scalar or vector) type.
func structArrayFieldFromProto(p *schemapb.StructArrayFieldSchema) *Field {
structSchema := NewStructSchema()
typeParams := KvPairsMap(p.GetTypeParams())
for _, sf := range p.GetFields() {
field := NewField().ReadProto(sf)
// unwrap Array/ArrayOfVector wrapper added by ProtoMessage()
@@ -230,6 +232,11 @@ func structArrayFieldFromProto(p *schemapb.StructArrayFieldSchema) *Field {
field.DataType = FieldType(sf.GetElementType())
field.ElementType = 0
}
if _, ok := typeParams[TypeParamMaxCapacity]; !ok {
if maxCapacity, has := field.TypeParams[TypeParamMaxCapacity]; has {
typeParams[TypeParamMaxCapacity] = maxCapacity
}
}
structSchema.WithField(field)
}
return &Field{
@@ -238,7 +245,8 @@ func structArrayFieldFromProto(p *schemapb.StructArrayFieldSchema) *Field {
Description: p.GetDescription(),
DataType: FieldTypeArray,
ElementType: FieldTypeStruct,
TypeParams: KvPairsMap(p.GetTypeParams()),
TypeParams: typeParams,
Nullable: p.GetNullable(),
StructSchema: structSchema,
}
}
+8
View File
@@ -214,11 +214,17 @@ func (s *SchemaSuite) TestStructArrayFieldRoundTrip() {
WithDataType(FieldTypeArray).
WithElementType(FieldTypeStruct).
WithMaxCapacity(16).
WithNullable(true).
WithStructSchema(structSchema))
p := schema.ProtoMessage()
s.Equal(2, len(p.GetFields()))
s.Equal(1, len(p.GetStructArrayFields()))
s.True(p.GetStructArrayFields()[0].GetNullable())
s.Equal("16", KvPairsMap(p.GetStructArrayFields()[0].GetTypeParams())[TypeParamMaxCapacity])
// DescribeCollection may return max_capacity only on struct sub-fields.
p.GetStructArrayFields()[0].TypeParams = nil
got := (&Schema{}).ReadProto(p)
// 3 logical fields including the struct array
@@ -234,6 +240,8 @@ func (s *SchemaSuite) TestStructArrayFieldRoundTrip() {
s.Require().NotNil(clips)
s.Equal(FieldTypeArray, clips.DataType)
s.Equal(FieldTypeStruct, clips.ElementType)
s.True(clips.Nullable)
s.Equal("16", clips.TypeParams[TypeParamMaxCapacity])
s.Require().NotNil(clips.StructSchema)
s.Equal(2, len(clips.StructSchema.Fields))
+15
View File
@@ -211,3 +211,18 @@ func (c *Client) AddCollectionField(ctx context.Context, opt AddCollectionFieldO
})
return err
}
// AddCollectionStructField adds a struct array field to a collection.
func (c *Client) AddCollectionStructField(ctx context.Context, opt AddCollectionStructFieldOption, callOpts ...grpc.CallOption) error {
if err := opt.Validate(); err != nil {
return err
}
req := opt.Request()
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
resp, err := milvusService.AddCollectionStructField(ctx, req, callOpts...)
return merr.CheckRPCCall(resp, err)
})
return err
}
+43
View File
@@ -465,3 +465,46 @@ func NewAddCollectionFieldOption(collectionName string, field *entity.Field) *ad
fieldSch: field,
}
}
type AddCollectionStructFieldOption interface {
Request() *milvuspb.AddCollectionStructFieldRequest
// Validate validates the option before sending request
Validate() error
}
type addCollectionStructFieldOption struct {
collectionName string
fieldSch *entity.Field
}
func (c *addCollectionStructFieldOption) Request() *milvuspb.AddCollectionStructFieldRequest {
collSchema := entity.NewSchema().WithField(c.fieldSch).ProtoMessage()
return &milvuspb.AddCollectionStructFieldRequest{
CollectionName: c.collectionName,
StructArrayFieldSchema: collSchema.GetStructArrayFields()[0],
}
}
// Validate validates the option before sending request
func (c *addCollectionStructFieldOption) Validate() error {
if c == nil {
return fmt.Errorf("add collection struct field option is nil")
}
if c.fieldSch == nil {
return fmt.Errorf("struct array field schema is required")
}
if c.fieldSch.DataType != entity.FieldTypeArray || c.fieldSch.ElementType != entity.FieldTypeStruct {
return fmt.Errorf("adding struct array field requires data type Array and element type Struct, field name = %s", c.fieldSch.Name)
}
if c.fieldSch.StructSchema == nil {
return fmt.Errorf("struct schema is required for struct array field %s", c.fieldSch.Name)
}
return c.fieldSch.StructSchema.Validate(c.fieldSch.Name)
}
func NewAddCollectionStructFieldOption(collectionName string, field *entity.Field) *addCollectionStructFieldOption {
return &addCollectionStructFieldOption{
collectionName: collectionName,
fieldSch: field,
}
}
+114
View File
@@ -494,6 +494,120 @@ func (s *CollectionSuite) TestAddCollectionField() {
})
}
func (s *CollectionSuite) TestAddCollectionStructField() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s.Run("success", func() {
collName := fmt.Sprintf("coll_%s", s.randString(6))
fieldName := fmt.Sprintf("field_%s", s.randString(6))
s.mock.EXPECT().AddCollectionStructField(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, acsfr *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
s.Equal(collName, acsfr.GetCollectionName())
structProto := acsfr.GetStructArrayFieldSchema()
s.Require().NotNil(structProto)
s.Equal(fieldName, structProto.GetName())
s.True(structProto.GetNullable())
s.Equal("16", entity.KvPairsMap(structProto.GetTypeParams())[common.MaxCapacityKey])
s.Require().Len(structProto.GetFields(), 2)
tagField := structProto.GetFields()[0]
s.Equal("tag", tagField.GetName())
s.Equal(schemapb.DataType_Array, tagField.GetDataType())
s.Equal(schemapb.DataType_VarChar, tagField.GetElementType())
tagParams := entity.KvPairsMap(tagField.GetTypeParams())
s.Equal("64", tagParams[common.MaxLengthKey])
s.Equal("16", tagParams[common.MaxCapacityKey])
embField := structProto.GetFields()[1]
s.Equal("emb", embField.GetName())
s.Equal(schemapb.DataType_ArrayOfVector, embField.GetDataType())
s.Equal(schemapb.DataType_FloatVector, embField.GetElementType())
embParams := entity.KvPairsMap(embField.GetTypeParams())
s.Equal("8", embParams[common.DimKey])
s.Equal("16", embParams[common.MaxCapacityKey])
return merr.Success(), nil
}).Once()
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64)).
WithField(entity.NewField().WithName("emb").WithDataType(entity.FieldTypeFloatVector).WithDim(8))
field := entity.NewField().
WithName(fieldName).
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(16).
WithNullable(true).
WithStructSchema(structSchema)
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
s.NoError(err)
})
s.Run("failure", func() {
collName := fmt.Sprintf("coll_%s", s.randString(6))
fieldName := fmt.Sprintf("field_%s", s.randString(6))
s.mock.EXPECT().AddCollectionStructField(mock.Anything, mock.Anything).Return(merr.Status(errors.New("mocked")), nil).Once()
field := entity.NewField().
WithName(fieldName).
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(16).
WithStructSchema(entity.NewStructSchema().WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64)))
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
s.Error(err)
})
s.Run("non_struct_array_field", func() {
collName := fmt.Sprintf("coll_%s", s.randString(6))
field := entity.NewField().WithName("field").WithDataType(entity.FieldTypeInt64).WithNullable(true)
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
s.Error(err)
s.Contains(err.Error(), "requires data type Array and element type Struct")
})
s.Run("nil_option", func() {
var opt *addCollectionStructFieldOption
err := opt.Validate()
s.Error(err)
s.Contains(err.Error(), "option is nil")
})
s.Run("nil_field_schema", func() {
collName := fmt.Sprintf("coll_%s", s.randString(6))
err := NewAddCollectionStructFieldOption(collName, nil).Validate()
s.Error(err)
s.Contains(err.Error(), "struct array field schema is required")
})
s.Run("nil_struct_schema", func() {
collName := fmt.Sprintf("coll_%s", s.randString(6))
field := entity.NewField().
WithName("clips").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct)
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
s.Error(err)
s.Contains(err.Error(), "struct schema is required")
})
s.Run("invalid_sub_field", func() {
collName := fmt.Sprintf("coll_%s", s.randString(6))
field := entity.NewField().
WithName("clips").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithStructSchema(entity.NewStructSchema().WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithNullable(true)))
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
s.Error(err)
s.Contains(err.Error(), "must not be nullable")
})
}
func TestCollection(t *testing.T) {
suite.Run(t, new(CollectionSuite))
}
@@ -37,6 +37,7 @@ const (
ExternalCollectionJobCategory = "/jobs/external_collection/"
PrivilegeGroupCategory = "/privilege_groups/"
CollectionFieldCategory = "/collections/fields/"
CollectionStructFieldCategory = "/collections/struct_fields/"
ResourceGroupCategory = "/resource_groups/"
SegmentCategory = "/segments/"
QuotaCenterCategory = "/quotacenter/"
@@ -95,6 +95,7 @@ var routeToMethod = map[string]string{ //nolint:gosec // not credentials, just a
"/v2/vectordb/collections/fields/alter_properties": "AlterCollectionField",
"/v2/vectordb/collections/fields/add": "AddCollectionField",
"/v2/vectordb/collections/struct_fields/add": "AddCollectionStructField",
"/v2/vectordb/databases/create": "CreateDatabase",
"/v2/vectordb/databases/drop": "DropDatabase",
@@ -207,6 +208,7 @@ func (h *HandlersV2) RegisterRoutesToV2(router gin.IRouter) {
// /collections/fields/add
router.POST(CollectionFieldCategory+AddAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionFieldReqWithSchema{} }, wrapperTraceLog(h.addCollectionField))))
router.POST(CollectionStructFieldCategory+AddAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionFieldReqWithSchema{} }, wrapperTraceLog(h.addCollectionStructField))))
router.POST(DataBaseCategory+CreateAction, timeoutMiddleware(wrapperPost(func() any { return &DatabaseReqWithProperties{} }, wrapperTraceLog(h.createDatabase))))
router.POST(DataBaseCategory+DropAction, timeoutMiddleware(wrapperPost(func() any { return &DatabaseReqRequiredName{} }, wrapperTraceLog(h.dropDatabase))))
@@ -1077,6 +1079,12 @@ func (h *HandlersV2) alterCollectionFieldProperties(ctx context.Context, c *gin.
func (h *HandlersV2) addCollectionField(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
httpReq := anyReq.(*CollectionFieldReqWithSchema)
if httpReq.Schema.IsStructArrayField() {
err := merr.WrapErrParameterInvalidMsg("StructArray field must be added through /v2/vectordb/collections/struct_fields/add")
HTTPAbortReturn(c, http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
return nil, err
}
schemaProto, err := httpReq.Schema.GetProto(ctx)
if err != nil {
HTTPAbortReturn(c, http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
@@ -1105,6 +1113,32 @@ func (h *HandlersV2) addCollectionField(ctx context.Context, c *gin.Context, any
return resp, err
}
func (h *HandlersV2) addCollectionStructField(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
httpReq := anyReq.(*CollectionFieldReqWithSchema)
schemaProto, err := httpReq.Schema.GetStructArrayProto(ctx)
if err != nil {
HTTPAbortReturn(c, http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
return nil, err
}
req := &milvuspb.AddCollectionStructFieldRequest{
DbName: dbName,
CollectionName: httpReq.CollectionName,
StructArrayFieldSchema: schemaProto,
}
c.Set(ContextRequest, req)
resp, err := wrapperProxyWithLimit(ctx, c, req, h.checkAuth, false, "/milvus.proto.milvus.MilvusService/AddCollectionStructField", true, h.proxy, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.AddCollectionStructField(reqCtx, req.(*milvuspb.AddCollectionStructFieldRequest))
})
if err == nil {
HTTPReturn(c, http.StatusOK, wrapperReturnDefault())
}
return resp, err
}
// copy from internal/proxy/task_query.go
func matchCountRule(outputs []string) bool {
return len(outputs) == 1 && strings.ToLower(strings.TrimSpace(outputs[0])) == "count(*)"
@@ -3754,6 +3754,169 @@ func (s *AddCollectionFieldSuite) TestAddCollectionFieldNormal() {
validateRequestBodyTestCases(s.T(), s.testEngine, addFieldTestCases, false)
}
func (s *AddCollectionFieldSuite) TestAddCollectionStructFieldNormal() {
testCases := []struct {
name string
requestBody []byte
}{
{
name: "array_with_struct_element",
requestBody: []byte(`{"collectionName": "book", "schema": {
"fieldName": "clips",
"description": "clip metadata",
"dataType": "Array",
"elementDataType": "Struct",
"nullable": true,
"elementTypeParams": {"max_capacity": 16},
"fields": [
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}},
{"fieldName": "emb", "dataType": "ArrayOfVector", "elementDataType": "FloatVector", "elementTypeParams": {"dim": 8}}
]
}}`),
},
{
name: "array_of_struct",
requestBody: []byte(`{"collectionName": "book", "schema": {
"fieldName": "clips",
"dataType": "ArrayOfStruct",
"nullable": true,
"typeParams": {"max_capacity": 16},
"fields": [
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}},
{"fieldName": "emb", "dataType": "ArrayOfVector", "elementDataType": "FloatVector", "elementTypeParams": {"dim": 8}}
]
}}`),
},
}
for _, tc := range testCases {
s.Run(tc.name, func() {
s.mp.EXPECT().AddCollectionStructField(mock.Anything, mock.MatchedBy(func(req *milvuspb.AddCollectionStructFieldRequest) bool {
if req.GetCollectionName() != "book" {
return false
}
structSchema := req.GetStructArrayFieldSchema()
if structSchema.GetName() != "clips" || !structSchema.GetNullable() || len(structSchema.GetFields()) != 2 {
return false
}
if kvPairsToMap(structSchema.GetTypeParams())[common.MaxCapacityKey] != "16" {
return false
}
tag := structSchema.GetFields()[0]
tagParams := kvPairsToMap(tag.GetTypeParams())
if tag.GetName() != "tag" || tag.GetDataType() != schemapb.DataType_Array || tag.GetElementType() != schemapb.DataType_VarChar {
return false
}
if tagParams[common.MaxLengthKey] != "64" || tagParams[common.MaxCapacityKey] != "16" {
return false
}
emb := structSchema.GetFields()[1]
embParams := kvPairsToMap(emb.GetTypeParams())
if emb.GetName() != "emb" || emb.GetDataType() != schemapb.DataType_ArrayOfVector || emb.GetElementType() != schemapb.DataType_FloatVector {
return false
}
return embParams[common.DimKey] == "8" && embParams[common.MaxCapacityKey] == "16"
})).Return(merr.Success(), nil).Once()
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(tc.requestBody))
w := httptest.NewRecorder()
s.testEngine.ServeHTTP(w, req)
s.Equal(http.StatusOK, w.Code)
returnBody := &ReturnErrMsg{}
err := json.Unmarshal(w.Body.Bytes(), returnBody)
s.NoError(err)
s.Equal(int32(0), returnBody.Code)
})
}
}
func (s *AddCollectionFieldSuite) TestAddCollectionStructFieldFail() {
s.Run("reject_non_struct_array_schema", func() {
requestBody := []byte(`{"collectionName": "book", "schema": {
"fieldName": "bad_scalar",
"dataType": "Int64",
"fields": [
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}}
]
}}`)
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(requestBody))
w := httptest.NewRecorder()
s.testEngine.ServeHTTP(w, req)
s.Equal(http.StatusOK, w.Code)
returnBody := &ReturnErrMsg{}
err := json.Unmarshal(w.Body.Bytes(), returnBody)
s.NoError(err)
s.Equal(merr.Code(merr.ErrParameterInvalid), returnBody.Code)
s.Contains(returnBody.Message, "must use ArrayOfStruct or Array with Struct element")
})
s.Run("invalid_struct_schema", func() {
requestBody := []byte(`{"collectionName": "book", "schema": {
"fieldName": "clips",
"dataType": "ArrayOfStruct",
"nullable": true,
"typeParams": {"max_capacity": 16}
}}`)
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(requestBody))
w := httptest.NewRecorder()
s.testEngine.ServeHTTP(w, req)
s.Equal(http.StatusOK, w.Code)
returnBody := &ReturnErrMsg{}
err := json.Unmarshal(w.Body.Bytes(), returnBody)
s.NoError(err)
s.Equal(merr.Code(merr.ErrParameterInvalid), returnBody.Code)
s.Contains(returnBody.Message, "must contain at least one sub-field")
})
s.Run("server_error", func() {
requestBody := []byte(`{"collectionName": "book", "schema": {
"fieldName": "clips",
"dataType": "ArrayOfStruct",
"nullable": true,
"typeParams": {"max_capacity": 16},
"fields": [
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}}
]
}}`)
s.mp.EXPECT().AddCollectionStructField(mock.Anything, mock.Anything).Return(merr.Status(merr.WrapErrServiceInternal("mock error")), nil).Once()
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(requestBody))
w := httptest.NewRecorder()
s.testEngine.ServeHTTP(w, req)
s.Equal(http.StatusOK, w.Code)
returnBody := &ReturnErrMsg{}
err := json.Unmarshal(w.Body.Bytes(), returnBody)
s.NoError(err)
s.Equal(merr.Code(merr.ErrServiceInternal), returnBody.Code)
s.Contains(returnBody.Message, "mock error")
})
}
func (s *AddCollectionFieldSuite) TestAddCollectionFieldRejectsStructArray() {
requestBody := []byte(`{"collectionName": "book", "schema": {
"fieldName": "clips",
"dataType": "ArrayOfStruct",
"nullable": true,
"typeParams": {"max_capacity": 16},
"fields": [
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}}
]
}}`)
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionFieldCategory, AddAction), bytes.NewReader(requestBody))
w := httptest.NewRecorder()
s.testEngine.ServeHTTP(w, req)
s.Equal(http.StatusOK, w.Code)
returnBody := &ReturnErrMsg{}
err := json.Unmarshal(w.Body.Bytes(), returnBody)
s.NoError(err)
s.Equal(merr.Code(merr.ErrParameterInvalid), returnBody.Code)
s.Contains(returnBody.Message, "/v2/vectordb/collections/struct_fields/add")
}
func (s *AddCollectionFieldSuite) TestAddCollectionFieldFail() {
s.Run("bad_request", func() {
addFieldTestCases := []requestBodyTestCase{
@@ -3816,6 +3979,14 @@ func TestAddCollectionFieldSuite(t *testing.T) {
suite.Run(t, new(AddCollectionFieldSuite))
}
func kvPairsToMap(kvs []*commonpb.KeyValuePair) map[string]string {
ret := make(map[string]string, len(kvs))
for _, kv := range kvs {
ret[kv.GetKey()] = kv.GetValue()
}
return ret
}
func TestCollectionFunctionSuite(t *testing.T) {
suite.Run(t, new(CollectionFunctionSuite))
}
@@ -26,6 +26,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/log"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
)
@@ -642,6 +643,7 @@ func (req *DropIndexPropertiesReq) GetIndexName() string {
type FieldSchema struct {
FieldName string `json:"fieldName" binding:"required"`
Description string `json:"description"`
DataType string `json:"dataType" binding:"required"`
ElementDataType string `json:"elementDataType"`
ExternalField string `json:"externalField"`
@@ -649,6 +651,8 @@ type FieldSchema struct {
IsPartitionKey bool `json:"isPartitionKey"`
IsClusteringKey bool `json:"isClusteringKey"`
ElementTypeParams map[string]interface{} `json:"elementTypeParams"`
TypeParams map[string]interface{} `json:"typeParams"`
Fields []FieldSchema `json:"fields"`
Nullable bool `json:"nullable"`
DefaultValue interface{} `json:"defaultValue"`
}
@@ -662,6 +666,7 @@ func (field *FieldSchema) GetProto(ctx context.Context) (*schemapb.FieldSchema,
dataType := schemapb.DataType(fieldDataType)
fieldSchema := &schemapb.FieldSchema{
Name: field.FieldName,
Description: field.Description,
IsPrimaryKey: field.IsPrimary,
IsPartitionKey: field.IsPartitionKey,
IsClusteringKey: field.IsClusteringKey,
@@ -694,6 +699,39 @@ func (field *FieldSchema) GetProto(ctx context.Context) (*schemapb.FieldSchema,
return fieldSchema, nil
}
func (field *FieldSchema) IsStructArrayField() bool {
if field == nil {
return false
}
return field.DataType == schemapb.DataType_ArrayOfStruct.String() ||
field.DataType == schemapb.DataType_Array.String() && field.ElementDataType == schemapb.DataType_Struct.String()
}
func (field *FieldSchema) GetStructArrayProto(ctx context.Context) (*schemapb.StructArrayFieldSchema, error) {
if field == nil {
return nil, merr.WrapErrParameterInvalidMsg("StructArray field schema is required")
}
if !field.IsStructArrayField() {
return nil, merr.WrapErrParameterInvalidMsg(
"StructArray field must use ArrayOfStruct or Array with Struct element, got dataType %s and elementDataType %s",
field.DataType, field.ElementDataType)
}
typeParams := make(map[string]interface{}, len(field.TypeParams)+len(field.ElementTypeParams))
for key, param := range field.ElementTypeParams {
typeParams[key] = param
}
for key, param := range field.TypeParams {
typeParams[key] = param
}
return (&StructArrayFieldSchema{
FieldName: field.FieldName,
Description: field.Description,
Fields: field.Fields,
TypeParams: typeParams,
Nullable: field.Nullable,
}).GetProto(ctx)
}
// StructArrayFieldSchema describes a struct array field in RESTful v2 API.
// Each struct array field contains multiple sub-fields; every sub-field must
// be declared as either Array (scalar element) or ArrayOfVector (vector element).
@@ -702,6 +740,7 @@ type StructArrayFieldSchema struct {
Description string `json:"description"`
Fields []FieldSchema `json:"fields" binding:"required"`
TypeParams map[string]interface{} `json:"typeParams"`
Nullable bool `json:"nullable"`
}
// GetProto converts the RESTful StructArrayFieldSchema to its proto counterpart.
@@ -714,6 +753,16 @@ func (sf *StructArrayFieldSchema) GetProto(ctx context.Context) (*schemapb.Struc
Name: sf.FieldName,
Description: sf.Description,
TypeParams: []*commonpb.KeyValuePair{},
Nullable: sf.Nullable,
}
parentTypeParams := make(map[string]string, len(sf.TypeParams))
for key, param := range sf.TypeParams {
value, err := getElementTypeParams(param)
if err != nil {
return nil, err
}
parentTypeParams[key] = value
proto.TypeParams = append(proto.TypeParams, &commonpb.KeyValuePair{Key: key, Value: value})
}
subNames := map[string]struct{}{}
for i := range sf.Fields {
@@ -747,18 +796,23 @@ func (sf *StructArrayFieldSchema) GetProto(ctx context.Context) (*schemapb.Struc
"duplicated sub-field name %s in struct %s", subProto.Name, sf.FieldName)
}
subNames[subProto.Name] = struct{}{}
if maxCapacity, ok := parentTypeParams[common.MaxCapacityKey]; ok && !hasTypeParam(subProto.TypeParams, common.MaxCapacityKey) {
subProto.TypeParams = append(subProto.TypeParams, &commonpb.KeyValuePair{Key: common.MaxCapacityKey, Value: maxCapacity})
}
proto.Fields = append(proto.Fields, subProto)
}
for key, param := range sf.TypeParams {
value, err := getElementTypeParams(param)
if err != nil {
return nil, err
}
proto.TypeParams = append(proto.TypeParams, &commonpb.KeyValuePair{Key: key, Value: value})
}
return proto, nil
}
func hasTypeParam(typeParams []*commonpb.KeyValuePair, key string) bool {
for _, typeParam := range typeParams {
if typeParam.GetKey() == key {
return true
}
}
return false
}
type FunctionScore struct {
Functions []FunctionSchema `json:"functions"`
Params map[string]interface{} `json:"params"`
@@ -408,11 +408,12 @@ func printStructArrayFieldsV2(structFields []*schemapb.StructArrayFieldSchema) [
subs = append(subs, detail)
}
entry := gin.H{
HTTPReturnFieldName: sf.GetName(),
HTTPReturnFieldID: sf.GetFieldID(),
HTTPReturnDescription: sf.GetDescription(),
HTTPReturnFieldType: schemapb.DataType_ArrayOfStruct.String(),
"fields": subs,
HTTPReturnFieldName: sf.GetName(),
HTTPReturnFieldID: sf.GetFieldID(),
HTTPReturnDescription: sf.GetDescription(),
HTTPReturnFieldNullable: sf.GetNullable(),
HTTPReturnFieldType: schemapb.DataType_ArrayOfStruct.String(),
"fields": subs,
}
if len(sf.GetTypeParams()) > 0 {
entry[Params] = sf.GetTypeParams()
@@ -4006,11 +4006,13 @@ func TestIsEmbeddingListData(t *testing.T) {
func TestPrintStructArrayFieldsV2(t *testing.T) {
schema := buildStructArrayTestSchema()
schema.GetStructArrayFields()[0].Nullable = true
printed := printStructArrayFieldsV2(schema.GetStructArrayFields())
require.Len(t, printed, 1)
entry := printed[0]
assert.Equal(t, "my_struct", entry[HTTPReturnFieldName])
assert.Equal(t, schemapb.DataType_ArrayOfStruct.String(), entry[HTTPReturnFieldType])
assert.Equal(t, true, entry[HTTPReturnFieldNullable])
subs, ok := entry["fields"].([]gin.H)
require.True(t, ok)
require.Len(t, subs, 2)
@@ -4508,6 +4510,7 @@ func TestStructArrayFieldSchemaGetProtoTypeParams(t *testing.T) {
proto, err := (&StructArrayFieldSchema{
FieldName: "my_struct",
Description: "with params",
Nullable: true,
TypeParams: map[string]interface{}{
common.MaxCapacityKey: 8,
},
@@ -4518,9 +4521,14 @@ func TestStructArrayFieldSchemaGetProtoTypeParams(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, "my_struct", proto.GetName())
assert.Equal(t, "with params", proto.GetDescription())
assert.True(t, proto.GetNullable())
require.Len(t, proto.GetTypeParams(), 1)
assert.Equal(t, common.MaxCapacityKey, proto.GetTypeParams()[0].GetKey())
assert.Equal(t, "8", proto.GetTypeParams()[0].GetValue())
subParams := proto.GetFields()[0].GetTypeParams()
require.Len(t, subParams, 1)
assert.Equal(t, common.MaxCapacityKey, subParams[0].GetKey())
assert.Equal(t, "8", subParams[0].GetValue())
_, err = (&StructArrayFieldSchema{
FieldName: "bad_params",
@@ -4534,6 +4542,59 @@ func TestStructArrayFieldSchemaGetProtoTypeParams(t *testing.T) {
assert.Error(t, err)
}
func TestFieldSchemaStructArrayHelpers(t *testing.T) {
var nilField *FieldSchema
assert.False(t, nilField.IsStructArrayField())
assert.False(t, (&FieldSchema{DataType: "Int64"}).IsStructArrayField())
assert.True(t, (&FieldSchema{DataType: "Array", ElementDataType: "Struct"}).IsStructArrayField())
assert.True(t, (&FieldSchema{DataType: "ArrayOfStruct"}).IsStructArrayField())
proto, err := (&FieldSchema{
FieldName: "clips",
Description: "clip metadata",
DataType: "Array",
ElementDataType: "Struct",
Nullable: true,
ElementTypeParams: map[string]interface{}{
common.MaxCapacityKey: 16,
},
TypeParams: map[string]interface{}{
common.MaxCapacityKey: 32,
},
Fields: []FieldSchema{
{
FieldName: "tag",
DataType: "Array",
ElementDataType: "VarChar",
ElementTypeParams: map[string]interface{}{
common.MaxLengthKey: 64,
},
},
{
FieldName: "scores",
DataType: "Array",
ElementDataType: "Int64",
ElementTypeParams: map[string]interface{}{
common.MaxCapacityKey: 7,
},
},
},
}).GetStructArrayProto(context.Background())
require.NoError(t, err)
assert.Equal(t, "clips", proto.GetName())
assert.Equal(t, "clip metadata", proto.GetDescription())
assert.True(t, proto.GetNullable())
assert.Equal(t, "32", kvPairsToMap(proto.GetTypeParams())[common.MaxCapacityKey])
require.Len(t, proto.GetFields(), 2)
tagParams := kvPairsToMap(proto.GetFields()[0].GetTypeParams())
assert.Equal(t, "64", tagParams[common.MaxLengthKey])
assert.Equal(t, "32", tagParams[common.MaxCapacityKey])
scoreParams := kvPairsToMap(proto.GetFields()[1].GetTypeParams())
assert.Equal(t, "7", scoreParams[common.MaxCapacityKey])
}
func TestPrintStructArrayFieldsV2QualifiedSubFields(t *testing.T) {
printed := printStructArrayFieldsV2([]*schemapb.StructArrayFieldSchema{
{
@@ -76,6 +76,60 @@ func TestAddCollectionField(t *testing.T) {
}
}
func TestAddCollectionStructField(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateMilvusClient(ctx, t, &client.ClientConfig{
Address: hp.GetAddr(),
Username: hp.GetUser(),
Password: hp.GetPassword(),
})
collName := common.GenRandomString("addstructfield", 6)
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim))
common.CheckErr(t, err, true)
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64)).
WithField(entity.NewField().WithName("embedding").WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim))
structField := entity.NewField().
WithName("clips").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(16).
WithNullable(true).
WithStructSchema(structSchema)
err = mc.AddCollectionStructField(ctx, client.NewAddCollectionStructFieldOption(collName, structField))
common.CheckErr(t, err, true)
coll, err := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
var added *entity.Field
for _, field := range coll.Schema.Fields {
if field.Name == "clips" {
added = field
break
}
}
require.NotNil(t, added)
require.Equal(t, entity.FieldTypeArray, added.DataType)
require.Equal(t, entity.FieldTypeStruct, added.ElementType)
require.True(t, added.Nullable)
require.Equal(t, "16", added.TypeParams[entity.TypeParamMaxCapacity])
require.NotNil(t, added.StructSchema)
require.Len(t, added.StructSchema.Fields, 2)
require.Equal(t, "tag", added.StructSchema.Fields[0].Name)
require.Equal(t, entity.FieldTypeVarChar, added.StructSchema.Fields[0].DataType)
require.Equal(t, "embedding", added.StructSchema.Fields[1].Name)
require.Equal(t, entity.FieldTypeFloatVector, added.StructSchema.Fields[1].DataType)
dim, err := added.StructSchema.Fields[1].GetDim()
common.CheckErr(t, err, true)
require.EqualValues(t, common.DefaultDim, dim)
}
// parameterized test for add field invalid cases
func TestAddCollectionFieldInvalid(t *testing.T) {
t.Parallel()
File diff suppressed because it is too large Load Diff
@@ -1792,6 +1792,131 @@ class TestCollectionAddField(TestBase):
assert rsp["code"] == 0
assert len(rsp["data"]) > 0
@pytest.mark.parametrize(
"schema_variant,field_params",
[
(
"array_with_struct_element",
{
"fieldName": "dynamic_struct",
"dataType": "Array",
"elementDataType": "Struct",
"nullable": True,
"elementTypeParams": {"max_capacity": 16},
"fields": [
{"fieldName": "sub_int", "dataType": "Array", "elementDataType": "Int32"},
{
"fieldName": "sub_vec",
"dataType": "ArrayOfVector",
"elementDataType": "FloatVector",
"elementTypeParams": {"dim": 8},
},
],
},
),
(
"array_of_struct",
{
"fieldName": "dynamic_struct",
"dataType": "ArrayOfStruct",
"nullable": True,
"typeParams": {"max_capacity": 16},
"fields": [
{"fieldName": "sub_int", "dataType": "Array", "elementDataType": "Int32"},
{
"fieldName": "sub_vec",
"dataType": "ArrayOfVector",
"elementDataType": "FloatVector",
"elementTypeParams": {"dim": 8},
},
],
},
),
],
)
def test_add_struct_array_field(self, schema_variant, field_params):
"""
target: test REST v2 add StructArray field
method: create collection, add StructArray field, insert and query rows with the new field
expected: StructArray field is added and usable through REST v2
"""
name = gen_collection_name()
dim = DEFAULT_STRUCT_ARRAY_DIM
field_name = field_params["fieldName"]
client = self.collection_client
vector_client = self.vector_client
payload = {
"collectionName": name,
"schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": True},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": f"{dim}"}},
]
},
"indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_index", "metricType": "L2"}],
}
rsp = client.collection_create(payload)
assert rsp["code"] == 0, rsp
client.wait_load_completed(collection_name=name, timeout=60)
rsp = vector_client.vector_insert(
{
"collectionName": name,
"data": [{"book_id": 0, "book_intro": gen_vector(dim=dim)}],
}
)
assert rsp["code"] == 0, rsp
rsp = client.add_struct_field(name, field_params)
logger.info(f"add struct array field response ({schema_variant}): {rsp}")
assert rsp["code"] == 0, rsp
rsp = client.collection_describe(name)
assert rsp["code"] == 0, rsp
struct_fields = {field["name"]: field for field in rsp["data"].get("structFields", [])}
assert field_name in struct_fields, rsp
struct_field = struct_fields[field_name]
assert struct_field["type"] == "ArrayOfStruct"
sub_fields = {field["name"]: field for field in struct_field.get("fields", [])}
assert sorted(sub_fields) == ["sub_int", "sub_vec"]
assert sub_fields["sub_int"]["type"] == "Array"
assert sub_fields["sub_int"]["elementType"] == "Int32"
assert sub_fields["sub_vec"]["type"] == "ArrayOfVector"
assert sub_fields["sub_vec"]["elementType"] == "FloatVector"
for sub_field in sub_fields.values():
sub_params = {param["key"]: param["value"] for param in sub_field.get("params", [])}
assert str(sub_params["max_capacity"]) == str(DEFAULT_STRUCT_ARRAY_SUB_CAPACITY)
new_row = {
"book_id": 1,
"book_intro": gen_vector(dim=dim),
field_name: [
{"sub_int": 11, "sub_vec": _rand_struct_array_vector(dim)},
{"sub_int": 12, "sub_vec": _rand_struct_array_vector(dim)},
],
}
rsp = vector_client.vector_insert({"collectionName": name, "data": [new_row]})
assert rsp["code"] == 0, rsp
rsp = vector_client.vector_query(
{
"collectionName": name,
"filter": "book_id == 1",
"outputFields": ["book_id", field_name],
"limit": 1,
}
)
assert rsp["code"] == 0, rsp
assert len(rsp["data"]) == 1
got = rsp["data"][0]
assert got["book_id"] == 1
assert len(got[field_name]) == len(new_row[field_name])
for actual, expected in zip(got[field_name], new_row[field_name]):
assert int(actual["sub_int"]) == expected["sub_int"]
np.testing.assert_allclose(actual["sub_vec"], expected["sub_vec"], rtol=1e-5, atol=1e-5)
@pytest.mark.L1
class TestCollectionAddFieldNegative(TestBase):