mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
enhance: add dynamic StructArray field APIs (#50276)
issue: #50235 ### What problem does this PR solve? This PR adds API support for dynamically adding StructArray fields to an existing collection, aligned with the existing Python SDK API shape. ### What is changed? - Add Go SDK `Client.AddCollectionStructField(...)` and `NewAddCollectionStructFieldOption(...)`. - Convert Go SDK StructArray field schema into `AddCollectionStructFieldRequest`. - Preserve StructArray parent `nullable` and `max_capacity` when converting schemas and describing collections. - Add REST v2 endpoint: - `POST /v2/vectordb/collections/struct_fields/add` - Reject StructArray payloads from the ordinary REST v2 field endpoint: - `POST /v2/vectordb/collections/fields/add` - Support REST v2 StructArray add payloads using both: - `dataType: "Array", elementDataType: "Struct"` - `dataType: "ArrayOfStruct"` - Add Go SDK and REST v2 regression coverage. ### Verification - `milvus-dev-cli go-ut local . -b master -t internal/distributed/proxy/httpserver -k 'TestAddCollectionFieldSuite' -f` - `ut-go-local-zhuwenxi-issue-50235-cbec6c2-26ef323b` - `milvus-dev-cli build local . -b master -f` - `build-local-zhuwenxing-issue-50235-clea-cbec6c2` - `milvus-dev-cli deploy create build-local-zhuwenxing-issue-50235-clea-cbec6c2 --name issue-50235-struct-add` - `pytest -q testcases/test_collection_operations.py::TestCollectionAddField::test_add_struct_array_field --endpoint http://10.100.36.203:19530 --token root:Milvus --minio_host 10.100.36.198` - `2 passed` --------- Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
@@ -154,6 +154,7 @@ func (s *Schema) ProtoMessage() *schemapb.CollectionSchema {
|
||||
Name: field.Name,
|
||||
Description: field.Description,
|
||||
TypeParams: MapKvPairs(field.TypeParams),
|
||||
Nullable: field.Nullable,
|
||||
}
|
||||
// max_capacity declared on the parent struct field must be carried onto each sub-field's
|
||||
// type params — the server validates it per sub-field on insert.
|
||||
@@ -222,6 +223,7 @@ func (s *Schema) ReadProto(p *schemapb.CollectionSchema) *Schema {
|
||||
// original (scalar or vector) type.
|
||||
func structArrayFieldFromProto(p *schemapb.StructArrayFieldSchema) *Field {
|
||||
structSchema := NewStructSchema()
|
||||
typeParams := KvPairsMap(p.GetTypeParams())
|
||||
for _, sf := range p.GetFields() {
|
||||
field := NewField().ReadProto(sf)
|
||||
// unwrap Array/ArrayOfVector wrapper added by ProtoMessage()
|
||||
@@ -230,6 +232,11 @@ func structArrayFieldFromProto(p *schemapb.StructArrayFieldSchema) *Field {
|
||||
field.DataType = FieldType(sf.GetElementType())
|
||||
field.ElementType = 0
|
||||
}
|
||||
if _, ok := typeParams[TypeParamMaxCapacity]; !ok {
|
||||
if maxCapacity, has := field.TypeParams[TypeParamMaxCapacity]; has {
|
||||
typeParams[TypeParamMaxCapacity] = maxCapacity
|
||||
}
|
||||
}
|
||||
structSchema.WithField(field)
|
||||
}
|
||||
return &Field{
|
||||
@@ -238,7 +245,8 @@ func structArrayFieldFromProto(p *schemapb.StructArrayFieldSchema) *Field {
|
||||
Description: p.GetDescription(),
|
||||
DataType: FieldTypeArray,
|
||||
ElementType: FieldTypeStruct,
|
||||
TypeParams: KvPairsMap(p.GetTypeParams()),
|
||||
TypeParams: typeParams,
|
||||
Nullable: p.GetNullable(),
|
||||
StructSchema: structSchema,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,11 +214,17 @@ func (s *SchemaSuite) TestStructArrayFieldRoundTrip() {
|
||||
WithDataType(FieldTypeArray).
|
||||
WithElementType(FieldTypeStruct).
|
||||
WithMaxCapacity(16).
|
||||
WithNullable(true).
|
||||
WithStructSchema(structSchema))
|
||||
|
||||
p := schema.ProtoMessage()
|
||||
s.Equal(2, len(p.GetFields()))
|
||||
s.Equal(1, len(p.GetStructArrayFields()))
|
||||
s.True(p.GetStructArrayFields()[0].GetNullable())
|
||||
s.Equal("16", KvPairsMap(p.GetStructArrayFields()[0].GetTypeParams())[TypeParamMaxCapacity])
|
||||
|
||||
// DescribeCollection may return max_capacity only on struct sub-fields.
|
||||
p.GetStructArrayFields()[0].TypeParams = nil
|
||||
|
||||
got := (&Schema{}).ReadProto(p)
|
||||
// 3 logical fields including the struct array
|
||||
@@ -234,6 +240,8 @@ func (s *SchemaSuite) TestStructArrayFieldRoundTrip() {
|
||||
s.Require().NotNil(clips)
|
||||
s.Equal(FieldTypeArray, clips.DataType)
|
||||
s.Equal(FieldTypeStruct, clips.ElementType)
|
||||
s.True(clips.Nullable)
|
||||
s.Equal("16", clips.TypeParams[TypeParamMaxCapacity])
|
||||
s.Require().NotNil(clips.StructSchema)
|
||||
s.Equal(2, len(clips.StructSchema.Fields))
|
||||
|
||||
|
||||
@@ -211,3 +211,18 @@ func (c *Client) AddCollectionField(ctx context.Context, opt AddCollectionFieldO
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// AddCollectionStructField adds a struct array field to a collection.
|
||||
func (c *Client) AddCollectionStructField(ctx context.Context, opt AddCollectionStructFieldOption, callOpts ...grpc.CallOption) error {
|
||||
if err := opt.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req := opt.Request()
|
||||
|
||||
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
||||
resp, err := milvusService.AddCollectionStructField(ctx, req, callOpts...)
|
||||
return merr.CheckRPCCall(resp, err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -465,3 +465,46 @@ func NewAddCollectionFieldOption(collectionName string, field *entity.Field) *ad
|
||||
fieldSch: field,
|
||||
}
|
||||
}
|
||||
|
||||
type AddCollectionStructFieldOption interface {
|
||||
Request() *milvuspb.AddCollectionStructFieldRequest
|
||||
// Validate validates the option before sending request
|
||||
Validate() error
|
||||
}
|
||||
|
||||
type addCollectionStructFieldOption struct {
|
||||
collectionName string
|
||||
fieldSch *entity.Field
|
||||
}
|
||||
|
||||
func (c *addCollectionStructFieldOption) Request() *milvuspb.AddCollectionStructFieldRequest {
|
||||
collSchema := entity.NewSchema().WithField(c.fieldSch).ProtoMessage()
|
||||
return &milvuspb.AddCollectionStructFieldRequest{
|
||||
CollectionName: c.collectionName,
|
||||
StructArrayFieldSchema: collSchema.GetStructArrayFields()[0],
|
||||
}
|
||||
}
|
||||
|
||||
// Validate validates the option before sending request
|
||||
func (c *addCollectionStructFieldOption) Validate() error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("add collection struct field option is nil")
|
||||
}
|
||||
if c.fieldSch == nil {
|
||||
return fmt.Errorf("struct array field schema is required")
|
||||
}
|
||||
if c.fieldSch.DataType != entity.FieldTypeArray || c.fieldSch.ElementType != entity.FieldTypeStruct {
|
||||
return fmt.Errorf("adding struct array field requires data type Array and element type Struct, field name = %s", c.fieldSch.Name)
|
||||
}
|
||||
if c.fieldSch.StructSchema == nil {
|
||||
return fmt.Errorf("struct schema is required for struct array field %s", c.fieldSch.Name)
|
||||
}
|
||||
return c.fieldSch.StructSchema.Validate(c.fieldSch.Name)
|
||||
}
|
||||
|
||||
func NewAddCollectionStructFieldOption(collectionName string, field *entity.Field) *addCollectionStructFieldOption {
|
||||
return &addCollectionStructFieldOption{
|
||||
collectionName: collectionName,
|
||||
fieldSch: field,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -494,6 +494,120 @@ func (s *CollectionSuite) TestAddCollectionField() {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *CollectionSuite) TestAddCollectionStructField() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
s.Run("success", func() {
|
||||
collName := fmt.Sprintf("coll_%s", s.randString(6))
|
||||
fieldName := fmt.Sprintf("field_%s", s.randString(6))
|
||||
s.mock.EXPECT().AddCollectionStructField(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, acsfr *milvuspb.AddCollectionStructFieldRequest) (*commonpb.Status, error) {
|
||||
s.Equal(collName, acsfr.GetCollectionName())
|
||||
structProto := acsfr.GetStructArrayFieldSchema()
|
||||
s.Require().NotNil(structProto)
|
||||
s.Equal(fieldName, structProto.GetName())
|
||||
s.True(structProto.GetNullable())
|
||||
s.Equal("16", entity.KvPairsMap(structProto.GetTypeParams())[common.MaxCapacityKey])
|
||||
s.Require().Len(structProto.GetFields(), 2)
|
||||
|
||||
tagField := structProto.GetFields()[0]
|
||||
s.Equal("tag", tagField.GetName())
|
||||
s.Equal(schemapb.DataType_Array, tagField.GetDataType())
|
||||
s.Equal(schemapb.DataType_VarChar, tagField.GetElementType())
|
||||
tagParams := entity.KvPairsMap(tagField.GetTypeParams())
|
||||
s.Equal("64", tagParams[common.MaxLengthKey])
|
||||
s.Equal("16", tagParams[common.MaxCapacityKey])
|
||||
|
||||
embField := structProto.GetFields()[1]
|
||||
s.Equal("emb", embField.GetName())
|
||||
s.Equal(schemapb.DataType_ArrayOfVector, embField.GetDataType())
|
||||
s.Equal(schemapb.DataType_FloatVector, embField.GetElementType())
|
||||
embParams := entity.KvPairsMap(embField.GetTypeParams())
|
||||
s.Equal("8", embParams[common.DimKey])
|
||||
s.Equal("16", embParams[common.MaxCapacityKey])
|
||||
return merr.Success(), nil
|
||||
}).Once()
|
||||
|
||||
structSchema := entity.NewStructSchema().
|
||||
WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64)).
|
||||
WithField(entity.NewField().WithName("emb").WithDataType(entity.FieldTypeFloatVector).WithDim(8))
|
||||
field := entity.NewField().
|
||||
WithName(fieldName).
|
||||
WithDataType(entity.FieldTypeArray).
|
||||
WithElementType(entity.FieldTypeStruct).
|
||||
WithMaxCapacity(16).
|
||||
WithNullable(true).
|
||||
WithStructSchema(structSchema)
|
||||
|
||||
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
|
||||
s.NoError(err)
|
||||
})
|
||||
|
||||
s.Run("failure", func() {
|
||||
collName := fmt.Sprintf("coll_%s", s.randString(6))
|
||||
fieldName := fmt.Sprintf("field_%s", s.randString(6))
|
||||
s.mock.EXPECT().AddCollectionStructField(mock.Anything, mock.Anything).Return(merr.Status(errors.New("mocked")), nil).Once()
|
||||
|
||||
field := entity.NewField().
|
||||
WithName(fieldName).
|
||||
WithDataType(entity.FieldTypeArray).
|
||||
WithElementType(entity.FieldTypeStruct).
|
||||
WithMaxCapacity(16).
|
||||
WithStructSchema(entity.NewStructSchema().WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64)))
|
||||
|
||||
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
|
||||
s.Error(err)
|
||||
})
|
||||
|
||||
s.Run("non_struct_array_field", func() {
|
||||
collName := fmt.Sprintf("coll_%s", s.randString(6))
|
||||
field := entity.NewField().WithName("field").WithDataType(entity.FieldTypeInt64).WithNullable(true)
|
||||
|
||||
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
|
||||
s.Error(err)
|
||||
s.Contains(err.Error(), "requires data type Array and element type Struct")
|
||||
})
|
||||
|
||||
s.Run("nil_option", func() {
|
||||
var opt *addCollectionStructFieldOption
|
||||
err := opt.Validate()
|
||||
s.Error(err)
|
||||
s.Contains(err.Error(), "option is nil")
|
||||
})
|
||||
|
||||
s.Run("nil_field_schema", func() {
|
||||
collName := fmt.Sprintf("coll_%s", s.randString(6))
|
||||
err := NewAddCollectionStructFieldOption(collName, nil).Validate()
|
||||
s.Error(err)
|
||||
s.Contains(err.Error(), "struct array field schema is required")
|
||||
})
|
||||
|
||||
s.Run("nil_struct_schema", func() {
|
||||
collName := fmt.Sprintf("coll_%s", s.randString(6))
|
||||
field := entity.NewField().
|
||||
WithName("clips").
|
||||
WithDataType(entity.FieldTypeArray).
|
||||
WithElementType(entity.FieldTypeStruct)
|
||||
|
||||
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
|
||||
s.Error(err)
|
||||
s.Contains(err.Error(), "struct schema is required")
|
||||
})
|
||||
|
||||
s.Run("invalid_sub_field", func() {
|
||||
collName := fmt.Sprintf("coll_%s", s.randString(6))
|
||||
field := entity.NewField().
|
||||
WithName("clips").
|
||||
WithDataType(entity.FieldTypeArray).
|
||||
WithElementType(entity.FieldTypeStruct).
|
||||
WithStructSchema(entity.NewStructSchema().WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithNullable(true)))
|
||||
|
||||
err := s.client.AddCollectionStructField(ctx, NewAddCollectionStructFieldOption(collName, field))
|
||||
s.Error(err)
|
||||
s.Contains(err.Error(), "must not be nullable")
|
||||
})
|
||||
}
|
||||
|
||||
func TestCollection(t *testing.T) {
|
||||
suite.Run(t, new(CollectionSuite))
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ const (
|
||||
ExternalCollectionJobCategory = "/jobs/external_collection/"
|
||||
PrivilegeGroupCategory = "/privilege_groups/"
|
||||
CollectionFieldCategory = "/collections/fields/"
|
||||
CollectionStructFieldCategory = "/collections/struct_fields/"
|
||||
ResourceGroupCategory = "/resource_groups/"
|
||||
SegmentCategory = "/segments/"
|
||||
QuotaCenterCategory = "/quotacenter/"
|
||||
|
||||
@@ -95,6 +95,7 @@ var routeToMethod = map[string]string{ //nolint:gosec // not credentials, just a
|
||||
|
||||
"/v2/vectordb/collections/fields/alter_properties": "AlterCollectionField",
|
||||
"/v2/vectordb/collections/fields/add": "AddCollectionField",
|
||||
"/v2/vectordb/collections/struct_fields/add": "AddCollectionStructField",
|
||||
|
||||
"/v2/vectordb/databases/create": "CreateDatabase",
|
||||
"/v2/vectordb/databases/drop": "DropDatabase",
|
||||
@@ -207,6 +208,7 @@ func (h *HandlersV2) RegisterRoutesToV2(router gin.IRouter) {
|
||||
|
||||
// /collections/fields/add
|
||||
router.POST(CollectionFieldCategory+AddAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionFieldReqWithSchema{} }, wrapperTraceLog(h.addCollectionField))))
|
||||
router.POST(CollectionStructFieldCategory+AddAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionFieldReqWithSchema{} }, wrapperTraceLog(h.addCollectionStructField))))
|
||||
|
||||
router.POST(DataBaseCategory+CreateAction, timeoutMiddleware(wrapperPost(func() any { return &DatabaseReqWithProperties{} }, wrapperTraceLog(h.createDatabase))))
|
||||
router.POST(DataBaseCategory+DropAction, timeoutMiddleware(wrapperPost(func() any { return &DatabaseReqRequiredName{} }, wrapperTraceLog(h.dropDatabase))))
|
||||
@@ -1077,6 +1079,12 @@ func (h *HandlersV2) alterCollectionFieldProperties(ctx context.Context, c *gin.
|
||||
func (h *HandlersV2) addCollectionField(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
|
||||
httpReq := anyReq.(*CollectionFieldReqWithSchema)
|
||||
|
||||
if httpReq.Schema.IsStructArrayField() {
|
||||
err := merr.WrapErrParameterInvalidMsg("StructArray field must be added through /v2/vectordb/collections/struct_fields/add")
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
schemaProto, err := httpReq.Schema.GetProto(ctx)
|
||||
if err != nil {
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
|
||||
@@ -1105,6 +1113,32 @@ func (h *HandlersV2) addCollectionField(ctx context.Context, c *gin.Context, any
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (h *HandlersV2) addCollectionStructField(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
|
||||
httpReq := anyReq.(*CollectionFieldReqWithSchema)
|
||||
|
||||
schemaProto, err := httpReq.Schema.GetStructArrayProto(ctx)
|
||||
if err != nil {
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req := &milvuspb.AddCollectionStructFieldRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: httpReq.CollectionName,
|
||||
StructArrayFieldSchema: schemaProto,
|
||||
}
|
||||
|
||||
c.Set(ContextRequest, req)
|
||||
resp, err := wrapperProxyWithLimit(ctx, c, req, h.checkAuth, false, "/milvus.proto.milvus.MilvusService/AddCollectionStructField", true, h.proxy, func(reqCtx context.Context, req any) (interface{}, error) {
|
||||
return h.proxy.AddCollectionStructField(reqCtx, req.(*milvuspb.AddCollectionStructFieldRequest))
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
HTTPReturn(c, http.StatusOK, wrapperReturnDefault())
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// copy from internal/proxy/task_query.go
|
||||
func matchCountRule(outputs []string) bool {
|
||||
return len(outputs) == 1 && strings.ToLower(strings.TrimSpace(outputs[0])) == "count(*)"
|
||||
|
||||
@@ -3754,6 +3754,169 @@ func (s *AddCollectionFieldSuite) TestAddCollectionFieldNormal() {
|
||||
validateRequestBodyTestCases(s.T(), s.testEngine, addFieldTestCases, false)
|
||||
}
|
||||
|
||||
func (s *AddCollectionFieldSuite) TestAddCollectionStructFieldNormal() {
|
||||
testCases := []struct {
|
||||
name string
|
||||
requestBody []byte
|
||||
}{
|
||||
{
|
||||
name: "array_with_struct_element",
|
||||
requestBody: []byte(`{"collectionName": "book", "schema": {
|
||||
"fieldName": "clips",
|
||||
"description": "clip metadata",
|
||||
"dataType": "Array",
|
||||
"elementDataType": "Struct",
|
||||
"nullable": true,
|
||||
"elementTypeParams": {"max_capacity": 16},
|
||||
"fields": [
|
||||
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}},
|
||||
{"fieldName": "emb", "dataType": "ArrayOfVector", "elementDataType": "FloatVector", "elementTypeParams": {"dim": 8}}
|
||||
]
|
||||
}}`),
|
||||
},
|
||||
{
|
||||
name: "array_of_struct",
|
||||
requestBody: []byte(`{"collectionName": "book", "schema": {
|
||||
"fieldName": "clips",
|
||||
"dataType": "ArrayOfStruct",
|
||||
"nullable": true,
|
||||
"typeParams": {"max_capacity": 16},
|
||||
"fields": [
|
||||
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}},
|
||||
{"fieldName": "emb", "dataType": "ArrayOfVector", "elementDataType": "FloatVector", "elementTypeParams": {"dim": 8}}
|
||||
]
|
||||
}}`),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
s.Run(tc.name, func() {
|
||||
s.mp.EXPECT().AddCollectionStructField(mock.Anything, mock.MatchedBy(func(req *milvuspb.AddCollectionStructFieldRequest) bool {
|
||||
if req.GetCollectionName() != "book" {
|
||||
return false
|
||||
}
|
||||
structSchema := req.GetStructArrayFieldSchema()
|
||||
if structSchema.GetName() != "clips" || !structSchema.GetNullable() || len(structSchema.GetFields()) != 2 {
|
||||
return false
|
||||
}
|
||||
if kvPairsToMap(structSchema.GetTypeParams())[common.MaxCapacityKey] != "16" {
|
||||
return false
|
||||
}
|
||||
tag := structSchema.GetFields()[0]
|
||||
tagParams := kvPairsToMap(tag.GetTypeParams())
|
||||
if tag.GetName() != "tag" || tag.GetDataType() != schemapb.DataType_Array || tag.GetElementType() != schemapb.DataType_VarChar {
|
||||
return false
|
||||
}
|
||||
if tagParams[common.MaxLengthKey] != "64" || tagParams[common.MaxCapacityKey] != "16" {
|
||||
return false
|
||||
}
|
||||
emb := structSchema.GetFields()[1]
|
||||
embParams := kvPairsToMap(emb.GetTypeParams())
|
||||
if emb.GetName() != "emb" || emb.GetDataType() != schemapb.DataType_ArrayOfVector || emb.GetElementType() != schemapb.DataType_FloatVector {
|
||||
return false
|
||||
}
|
||||
return embParams[common.DimKey] == "8" && embParams[common.MaxCapacityKey] == "16"
|
||||
})).Return(merr.Success(), nil).Once()
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(tc.requestBody))
|
||||
w := httptest.NewRecorder()
|
||||
s.testEngine.ServeHTTP(w, req)
|
||||
s.Equal(http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
s.NoError(err)
|
||||
s.Equal(int32(0), returnBody.Code)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AddCollectionFieldSuite) TestAddCollectionStructFieldFail() {
|
||||
s.Run("reject_non_struct_array_schema", func() {
|
||||
requestBody := []byte(`{"collectionName": "book", "schema": {
|
||||
"fieldName": "bad_scalar",
|
||||
"dataType": "Int64",
|
||||
"fields": [
|
||||
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}}
|
||||
]
|
||||
}}`)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(requestBody))
|
||||
w := httptest.NewRecorder()
|
||||
s.testEngine.ServeHTTP(w, req)
|
||||
s.Equal(http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
s.NoError(err)
|
||||
s.Equal(merr.Code(merr.ErrParameterInvalid), returnBody.Code)
|
||||
s.Contains(returnBody.Message, "must use ArrayOfStruct or Array with Struct element")
|
||||
})
|
||||
|
||||
s.Run("invalid_struct_schema", func() {
|
||||
requestBody := []byte(`{"collectionName": "book", "schema": {
|
||||
"fieldName": "clips",
|
||||
"dataType": "ArrayOfStruct",
|
||||
"nullable": true,
|
||||
"typeParams": {"max_capacity": 16}
|
||||
}}`)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(requestBody))
|
||||
w := httptest.NewRecorder()
|
||||
s.testEngine.ServeHTTP(w, req)
|
||||
s.Equal(http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
s.NoError(err)
|
||||
s.Equal(merr.Code(merr.ErrParameterInvalid), returnBody.Code)
|
||||
s.Contains(returnBody.Message, "must contain at least one sub-field")
|
||||
})
|
||||
|
||||
s.Run("server_error", func() {
|
||||
requestBody := []byte(`{"collectionName": "book", "schema": {
|
||||
"fieldName": "clips",
|
||||
"dataType": "ArrayOfStruct",
|
||||
"nullable": true,
|
||||
"typeParams": {"max_capacity": 16},
|
||||
"fields": [
|
||||
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}}
|
||||
]
|
||||
}}`)
|
||||
|
||||
s.mp.EXPECT().AddCollectionStructField(mock.Anything, mock.Anything).Return(merr.Status(merr.WrapErrServiceInternal("mock error")), nil).Once()
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionStructFieldCategory, AddAction), bytes.NewReader(requestBody))
|
||||
w := httptest.NewRecorder()
|
||||
s.testEngine.ServeHTTP(w, req)
|
||||
s.Equal(http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
s.NoError(err)
|
||||
s.Equal(merr.Code(merr.ErrServiceInternal), returnBody.Code)
|
||||
s.Contains(returnBody.Message, "mock error")
|
||||
})
|
||||
}
|
||||
|
||||
func (s *AddCollectionFieldSuite) TestAddCollectionFieldRejectsStructArray() {
|
||||
requestBody := []byte(`{"collectionName": "book", "schema": {
|
||||
"fieldName": "clips",
|
||||
"dataType": "ArrayOfStruct",
|
||||
"nullable": true,
|
||||
"typeParams": {"max_capacity": 16},
|
||||
"fields": [
|
||||
{"fieldName": "tag", "dataType": "Array", "elementDataType": "VarChar", "elementTypeParams": {"max_length": 64}}
|
||||
]
|
||||
}}`)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, versionalV2(CollectionFieldCategory, AddAction), bytes.NewReader(requestBody))
|
||||
w := httptest.NewRecorder()
|
||||
s.testEngine.ServeHTTP(w, req)
|
||||
s.Equal(http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
s.NoError(err)
|
||||
s.Equal(merr.Code(merr.ErrParameterInvalid), returnBody.Code)
|
||||
s.Contains(returnBody.Message, "/v2/vectordb/collections/struct_fields/add")
|
||||
}
|
||||
|
||||
func (s *AddCollectionFieldSuite) TestAddCollectionFieldFail() {
|
||||
s.Run("bad_request", func() {
|
||||
addFieldTestCases := []requestBodyTestCase{
|
||||
@@ -3816,6 +3979,14 @@ func TestAddCollectionFieldSuite(t *testing.T) {
|
||||
suite.Run(t, new(AddCollectionFieldSuite))
|
||||
}
|
||||
|
||||
func kvPairsToMap(kvs []*commonpb.KeyValuePair) map[string]string {
|
||||
ret := make(map[string]string, len(kvs))
|
||||
for _, kv := range kvs {
|
||||
ret[kv.GetKey()] = kv.GetValue()
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func TestCollectionFunctionSuite(t *testing.T) {
|
||||
suite.Run(t, new(CollectionFunctionSuite))
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/log"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
)
|
||||
@@ -642,6 +643,7 @@ func (req *DropIndexPropertiesReq) GetIndexName() string {
|
||||
|
||||
type FieldSchema struct {
|
||||
FieldName string `json:"fieldName" binding:"required"`
|
||||
Description string `json:"description"`
|
||||
DataType string `json:"dataType" binding:"required"`
|
||||
ElementDataType string `json:"elementDataType"`
|
||||
ExternalField string `json:"externalField"`
|
||||
@@ -649,6 +651,8 @@ type FieldSchema struct {
|
||||
IsPartitionKey bool `json:"isPartitionKey"`
|
||||
IsClusteringKey bool `json:"isClusteringKey"`
|
||||
ElementTypeParams map[string]interface{} `json:"elementTypeParams"`
|
||||
TypeParams map[string]interface{} `json:"typeParams"`
|
||||
Fields []FieldSchema `json:"fields"`
|
||||
Nullable bool `json:"nullable"`
|
||||
DefaultValue interface{} `json:"defaultValue"`
|
||||
}
|
||||
@@ -662,6 +666,7 @@ func (field *FieldSchema) GetProto(ctx context.Context) (*schemapb.FieldSchema,
|
||||
dataType := schemapb.DataType(fieldDataType)
|
||||
fieldSchema := &schemapb.FieldSchema{
|
||||
Name: field.FieldName,
|
||||
Description: field.Description,
|
||||
IsPrimaryKey: field.IsPrimary,
|
||||
IsPartitionKey: field.IsPartitionKey,
|
||||
IsClusteringKey: field.IsClusteringKey,
|
||||
@@ -694,6 +699,39 @@ func (field *FieldSchema) GetProto(ctx context.Context) (*schemapb.FieldSchema,
|
||||
return fieldSchema, nil
|
||||
}
|
||||
|
||||
func (field *FieldSchema) IsStructArrayField() bool {
|
||||
if field == nil {
|
||||
return false
|
||||
}
|
||||
return field.DataType == schemapb.DataType_ArrayOfStruct.String() ||
|
||||
field.DataType == schemapb.DataType_Array.String() && field.ElementDataType == schemapb.DataType_Struct.String()
|
||||
}
|
||||
|
||||
func (field *FieldSchema) GetStructArrayProto(ctx context.Context) (*schemapb.StructArrayFieldSchema, error) {
|
||||
if field == nil {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("StructArray field schema is required")
|
||||
}
|
||||
if !field.IsStructArrayField() {
|
||||
return nil, merr.WrapErrParameterInvalidMsg(
|
||||
"StructArray field must use ArrayOfStruct or Array with Struct element, got dataType %s and elementDataType %s",
|
||||
field.DataType, field.ElementDataType)
|
||||
}
|
||||
typeParams := make(map[string]interface{}, len(field.TypeParams)+len(field.ElementTypeParams))
|
||||
for key, param := range field.ElementTypeParams {
|
||||
typeParams[key] = param
|
||||
}
|
||||
for key, param := range field.TypeParams {
|
||||
typeParams[key] = param
|
||||
}
|
||||
return (&StructArrayFieldSchema{
|
||||
FieldName: field.FieldName,
|
||||
Description: field.Description,
|
||||
Fields: field.Fields,
|
||||
TypeParams: typeParams,
|
||||
Nullable: field.Nullable,
|
||||
}).GetProto(ctx)
|
||||
}
|
||||
|
||||
// StructArrayFieldSchema describes a struct array field in RESTful v2 API.
|
||||
// Each struct array field contains multiple sub-fields; every sub-field must
|
||||
// be declared as either Array (scalar element) or ArrayOfVector (vector element).
|
||||
@@ -702,6 +740,7 @@ type StructArrayFieldSchema struct {
|
||||
Description string `json:"description"`
|
||||
Fields []FieldSchema `json:"fields" binding:"required"`
|
||||
TypeParams map[string]interface{} `json:"typeParams"`
|
||||
Nullable bool `json:"nullable"`
|
||||
}
|
||||
|
||||
// GetProto converts the RESTful StructArrayFieldSchema to its proto counterpart.
|
||||
@@ -714,6 +753,16 @@ func (sf *StructArrayFieldSchema) GetProto(ctx context.Context) (*schemapb.Struc
|
||||
Name: sf.FieldName,
|
||||
Description: sf.Description,
|
||||
TypeParams: []*commonpb.KeyValuePair{},
|
||||
Nullable: sf.Nullable,
|
||||
}
|
||||
parentTypeParams := make(map[string]string, len(sf.TypeParams))
|
||||
for key, param := range sf.TypeParams {
|
||||
value, err := getElementTypeParams(param)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parentTypeParams[key] = value
|
||||
proto.TypeParams = append(proto.TypeParams, &commonpb.KeyValuePair{Key: key, Value: value})
|
||||
}
|
||||
subNames := map[string]struct{}{}
|
||||
for i := range sf.Fields {
|
||||
@@ -747,18 +796,23 @@ func (sf *StructArrayFieldSchema) GetProto(ctx context.Context) (*schemapb.Struc
|
||||
"duplicated sub-field name %s in struct %s", subProto.Name, sf.FieldName)
|
||||
}
|
||||
subNames[subProto.Name] = struct{}{}
|
||||
if maxCapacity, ok := parentTypeParams[common.MaxCapacityKey]; ok && !hasTypeParam(subProto.TypeParams, common.MaxCapacityKey) {
|
||||
subProto.TypeParams = append(subProto.TypeParams, &commonpb.KeyValuePair{Key: common.MaxCapacityKey, Value: maxCapacity})
|
||||
}
|
||||
proto.Fields = append(proto.Fields, subProto)
|
||||
}
|
||||
for key, param := range sf.TypeParams {
|
||||
value, err := getElementTypeParams(param)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
proto.TypeParams = append(proto.TypeParams, &commonpb.KeyValuePair{Key: key, Value: value})
|
||||
}
|
||||
return proto, nil
|
||||
}
|
||||
|
||||
func hasTypeParam(typeParams []*commonpb.KeyValuePair, key string) bool {
|
||||
for _, typeParam := range typeParams {
|
||||
if typeParam.GetKey() == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type FunctionScore struct {
|
||||
Functions []FunctionSchema `json:"functions"`
|
||||
Params map[string]interface{} `json:"params"`
|
||||
|
||||
@@ -408,11 +408,12 @@ func printStructArrayFieldsV2(structFields []*schemapb.StructArrayFieldSchema) [
|
||||
subs = append(subs, detail)
|
||||
}
|
||||
entry := gin.H{
|
||||
HTTPReturnFieldName: sf.GetName(),
|
||||
HTTPReturnFieldID: sf.GetFieldID(),
|
||||
HTTPReturnDescription: sf.GetDescription(),
|
||||
HTTPReturnFieldType: schemapb.DataType_ArrayOfStruct.String(),
|
||||
"fields": subs,
|
||||
HTTPReturnFieldName: sf.GetName(),
|
||||
HTTPReturnFieldID: sf.GetFieldID(),
|
||||
HTTPReturnDescription: sf.GetDescription(),
|
||||
HTTPReturnFieldNullable: sf.GetNullable(),
|
||||
HTTPReturnFieldType: schemapb.DataType_ArrayOfStruct.String(),
|
||||
"fields": subs,
|
||||
}
|
||||
if len(sf.GetTypeParams()) > 0 {
|
||||
entry[Params] = sf.GetTypeParams()
|
||||
|
||||
@@ -4006,11 +4006,13 @@ func TestIsEmbeddingListData(t *testing.T) {
|
||||
|
||||
func TestPrintStructArrayFieldsV2(t *testing.T) {
|
||||
schema := buildStructArrayTestSchema()
|
||||
schema.GetStructArrayFields()[0].Nullable = true
|
||||
printed := printStructArrayFieldsV2(schema.GetStructArrayFields())
|
||||
require.Len(t, printed, 1)
|
||||
entry := printed[0]
|
||||
assert.Equal(t, "my_struct", entry[HTTPReturnFieldName])
|
||||
assert.Equal(t, schemapb.DataType_ArrayOfStruct.String(), entry[HTTPReturnFieldType])
|
||||
assert.Equal(t, true, entry[HTTPReturnFieldNullable])
|
||||
subs, ok := entry["fields"].([]gin.H)
|
||||
require.True(t, ok)
|
||||
require.Len(t, subs, 2)
|
||||
@@ -4508,6 +4510,7 @@ func TestStructArrayFieldSchemaGetProtoTypeParams(t *testing.T) {
|
||||
proto, err := (&StructArrayFieldSchema{
|
||||
FieldName: "my_struct",
|
||||
Description: "with params",
|
||||
Nullable: true,
|
||||
TypeParams: map[string]interface{}{
|
||||
common.MaxCapacityKey: 8,
|
||||
},
|
||||
@@ -4518,9 +4521,14 @@ func TestStructArrayFieldSchemaGetProtoTypeParams(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "my_struct", proto.GetName())
|
||||
assert.Equal(t, "with params", proto.GetDescription())
|
||||
assert.True(t, proto.GetNullable())
|
||||
require.Len(t, proto.GetTypeParams(), 1)
|
||||
assert.Equal(t, common.MaxCapacityKey, proto.GetTypeParams()[0].GetKey())
|
||||
assert.Equal(t, "8", proto.GetTypeParams()[0].GetValue())
|
||||
subParams := proto.GetFields()[0].GetTypeParams()
|
||||
require.Len(t, subParams, 1)
|
||||
assert.Equal(t, common.MaxCapacityKey, subParams[0].GetKey())
|
||||
assert.Equal(t, "8", subParams[0].GetValue())
|
||||
|
||||
_, err = (&StructArrayFieldSchema{
|
||||
FieldName: "bad_params",
|
||||
@@ -4534,6 +4542,59 @@ func TestStructArrayFieldSchemaGetProtoTypeParams(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestFieldSchemaStructArrayHelpers(t *testing.T) {
|
||||
var nilField *FieldSchema
|
||||
assert.False(t, nilField.IsStructArrayField())
|
||||
assert.False(t, (&FieldSchema{DataType: "Int64"}).IsStructArrayField())
|
||||
assert.True(t, (&FieldSchema{DataType: "Array", ElementDataType: "Struct"}).IsStructArrayField())
|
||||
assert.True(t, (&FieldSchema{DataType: "ArrayOfStruct"}).IsStructArrayField())
|
||||
|
||||
proto, err := (&FieldSchema{
|
||||
FieldName: "clips",
|
||||
Description: "clip metadata",
|
||||
DataType: "Array",
|
||||
ElementDataType: "Struct",
|
||||
Nullable: true,
|
||||
ElementTypeParams: map[string]interface{}{
|
||||
common.MaxCapacityKey: 16,
|
||||
},
|
||||
TypeParams: map[string]interface{}{
|
||||
common.MaxCapacityKey: 32,
|
||||
},
|
||||
Fields: []FieldSchema{
|
||||
{
|
||||
FieldName: "tag",
|
||||
DataType: "Array",
|
||||
ElementDataType: "VarChar",
|
||||
ElementTypeParams: map[string]interface{}{
|
||||
common.MaxLengthKey: 64,
|
||||
},
|
||||
},
|
||||
{
|
||||
FieldName: "scores",
|
||||
DataType: "Array",
|
||||
ElementDataType: "Int64",
|
||||
ElementTypeParams: map[string]interface{}{
|
||||
common.MaxCapacityKey: 7,
|
||||
},
|
||||
},
|
||||
},
|
||||
}).GetStructArrayProto(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "clips", proto.GetName())
|
||||
assert.Equal(t, "clip metadata", proto.GetDescription())
|
||||
assert.True(t, proto.GetNullable())
|
||||
assert.Equal(t, "32", kvPairsToMap(proto.GetTypeParams())[common.MaxCapacityKey])
|
||||
|
||||
require.Len(t, proto.GetFields(), 2)
|
||||
tagParams := kvPairsToMap(proto.GetFields()[0].GetTypeParams())
|
||||
assert.Equal(t, "64", tagParams[common.MaxLengthKey])
|
||||
assert.Equal(t, "32", tagParams[common.MaxCapacityKey])
|
||||
|
||||
scoreParams := kvPairsToMap(proto.GetFields()[1].GetTypeParams())
|
||||
assert.Equal(t, "7", scoreParams[common.MaxCapacityKey])
|
||||
}
|
||||
|
||||
func TestPrintStructArrayFieldsV2QualifiedSubFields(t *testing.T) {
|
||||
printed := printStructArrayFieldsV2([]*schemapb.StructArrayFieldSchema{
|
||||
{
|
||||
|
||||
@@ -76,6 +76,60 @@ func TestAddCollectionField(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddCollectionStructField(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := hp.CreateMilvusClient(ctx, t, &client.ClientConfig{
|
||||
Address: hp.GetAddr(),
|
||||
Username: hp.GetUser(),
|
||||
Password: hp.GetPassword(),
|
||||
})
|
||||
|
||||
collName := common.GenRandomString("addstructfield", 6)
|
||||
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
structSchema := entity.NewStructSchema().
|
||||
WithField(entity.NewField().WithName("tag").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64)).
|
||||
WithField(entity.NewField().WithName("embedding").WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim))
|
||||
structField := entity.NewField().
|
||||
WithName("clips").
|
||||
WithDataType(entity.FieldTypeArray).
|
||||
WithElementType(entity.FieldTypeStruct).
|
||||
WithMaxCapacity(16).
|
||||
WithNullable(true).
|
||||
WithStructSchema(structSchema)
|
||||
|
||||
err = mc.AddCollectionStructField(ctx, client.NewAddCollectionStructFieldOption(collName, structField))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
coll, err := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
var added *entity.Field
|
||||
for _, field := range coll.Schema.Fields {
|
||||
if field.Name == "clips" {
|
||||
added = field
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, added)
|
||||
require.Equal(t, entity.FieldTypeArray, added.DataType)
|
||||
require.Equal(t, entity.FieldTypeStruct, added.ElementType)
|
||||
require.True(t, added.Nullable)
|
||||
require.Equal(t, "16", added.TypeParams[entity.TypeParamMaxCapacity])
|
||||
require.NotNil(t, added.StructSchema)
|
||||
require.Len(t, added.StructSchema.Fields, 2)
|
||||
require.Equal(t, "tag", added.StructSchema.Fields[0].Name)
|
||||
require.Equal(t, entity.FieldTypeVarChar, added.StructSchema.Fields[0].DataType)
|
||||
require.Equal(t, "embedding", added.StructSchema.Fields[1].Name)
|
||||
require.Equal(t, entity.FieldTypeFloatVector, added.StructSchema.Fields[1].DataType)
|
||||
dim, err := added.StructSchema.Fields[1].GetDim()
|
||||
common.CheckErr(t, err, true)
|
||||
require.EqualValues(t, common.DefaultDim, dim)
|
||||
}
|
||||
|
||||
// parameterized test for add field invalid cases
|
||||
func TestAddCollectionFieldInvalid(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1792,6 +1792,131 @@ class TestCollectionAddField(TestBase):
|
||||
assert rsp["code"] == 0
|
||||
assert len(rsp["data"]) > 0
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema_variant,field_params",
|
||||
[
|
||||
(
|
||||
"array_with_struct_element",
|
||||
{
|
||||
"fieldName": "dynamic_struct",
|
||||
"dataType": "Array",
|
||||
"elementDataType": "Struct",
|
||||
"nullable": True,
|
||||
"elementTypeParams": {"max_capacity": 16},
|
||||
"fields": [
|
||||
{"fieldName": "sub_int", "dataType": "Array", "elementDataType": "Int32"},
|
||||
{
|
||||
"fieldName": "sub_vec",
|
||||
"dataType": "ArrayOfVector",
|
||||
"elementDataType": "FloatVector",
|
||||
"elementTypeParams": {"dim": 8},
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
"array_of_struct",
|
||||
{
|
||||
"fieldName": "dynamic_struct",
|
||||
"dataType": "ArrayOfStruct",
|
||||
"nullable": True,
|
||||
"typeParams": {"max_capacity": 16},
|
||||
"fields": [
|
||||
{"fieldName": "sub_int", "dataType": "Array", "elementDataType": "Int32"},
|
||||
{
|
||||
"fieldName": "sub_vec",
|
||||
"dataType": "ArrayOfVector",
|
||||
"elementDataType": "FloatVector",
|
||||
"elementTypeParams": {"dim": 8},
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_add_struct_array_field(self, schema_variant, field_params):
|
||||
"""
|
||||
target: test REST v2 add StructArray field
|
||||
method: create collection, add StructArray field, insert and query rows with the new field
|
||||
expected: StructArray field is added and usable through REST v2
|
||||
"""
|
||||
name = gen_collection_name()
|
||||
dim = DEFAULT_STRUCT_ARRAY_DIM
|
||||
field_name = field_params["fieldName"]
|
||||
client = self.collection_client
|
||||
vector_client = self.vector_client
|
||||
|
||||
payload = {
|
||||
"collectionName": name,
|
||||
"schema": {
|
||||
"fields": [
|
||||
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": True},
|
||||
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": f"{dim}"}},
|
||||
]
|
||||
},
|
||||
"indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_index", "metricType": "L2"}],
|
||||
}
|
||||
rsp = client.collection_create(payload)
|
||||
assert rsp["code"] == 0, rsp
|
||||
client.wait_load_completed(collection_name=name, timeout=60)
|
||||
|
||||
rsp = vector_client.vector_insert(
|
||||
{
|
||||
"collectionName": name,
|
||||
"data": [{"book_id": 0, "book_intro": gen_vector(dim=dim)}],
|
||||
}
|
||||
)
|
||||
assert rsp["code"] == 0, rsp
|
||||
|
||||
rsp = client.add_struct_field(name, field_params)
|
||||
logger.info(f"add struct array field response ({schema_variant}): {rsp}")
|
||||
assert rsp["code"] == 0, rsp
|
||||
|
||||
rsp = client.collection_describe(name)
|
||||
assert rsp["code"] == 0, rsp
|
||||
struct_fields = {field["name"]: field for field in rsp["data"].get("structFields", [])}
|
||||
assert field_name in struct_fields, rsp
|
||||
struct_field = struct_fields[field_name]
|
||||
assert struct_field["type"] == "ArrayOfStruct"
|
||||
|
||||
sub_fields = {field["name"]: field for field in struct_field.get("fields", [])}
|
||||
assert sorted(sub_fields) == ["sub_int", "sub_vec"]
|
||||
assert sub_fields["sub_int"]["type"] == "Array"
|
||||
assert sub_fields["sub_int"]["elementType"] == "Int32"
|
||||
assert sub_fields["sub_vec"]["type"] == "ArrayOfVector"
|
||||
assert sub_fields["sub_vec"]["elementType"] == "FloatVector"
|
||||
for sub_field in sub_fields.values():
|
||||
sub_params = {param["key"]: param["value"] for param in sub_field.get("params", [])}
|
||||
assert str(sub_params["max_capacity"]) == str(DEFAULT_STRUCT_ARRAY_SUB_CAPACITY)
|
||||
|
||||
new_row = {
|
||||
"book_id": 1,
|
||||
"book_intro": gen_vector(dim=dim),
|
||||
field_name: [
|
||||
{"sub_int": 11, "sub_vec": _rand_struct_array_vector(dim)},
|
||||
{"sub_int": 12, "sub_vec": _rand_struct_array_vector(dim)},
|
||||
],
|
||||
}
|
||||
rsp = vector_client.vector_insert({"collectionName": name, "data": [new_row]})
|
||||
assert rsp["code"] == 0, rsp
|
||||
|
||||
rsp = vector_client.vector_query(
|
||||
{
|
||||
"collectionName": name,
|
||||
"filter": "book_id == 1",
|
||||
"outputFields": ["book_id", field_name],
|
||||
"limit": 1,
|
||||
}
|
||||
)
|
||||
assert rsp["code"] == 0, rsp
|
||||
assert len(rsp["data"]) == 1
|
||||
got = rsp["data"][0]
|
||||
assert got["book_id"] == 1
|
||||
assert len(got[field_name]) == len(new_row[field_name])
|
||||
for actual, expected in zip(got[field_name], new_row[field_name]):
|
||||
assert int(actual["sub_int"]) == expected["sub_int"]
|
||||
np.testing.assert_allclose(actual["sub_vec"], expected["sub_vec"], rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.L1
|
||||
class TestCollectionAddFieldNegative(TestBase):
|
||||
|
||||
Reference in New Issue
Block a user