furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: [GoSDK] complete struct array support and correctness fixes (#49164)

Browse Source 
issue: #49163

## Summary

Brings the Go SDK (`client/v2`) struct array feature to parity with
pymilvus and fixes several client-side correctness issues found in the
master surface.

**New capabilities**

- Vector-array column types for struct sub-fields:
`ColumnFloatVectorArray`, `ColumnFloat16VectorArray`,
`ColumnBFloat16VectorArray`, `ColumnBinaryVectorArray`,
`ColumnInt8VectorArray` (new `client/column/vector_array.go`). Response
decoding goes through a shared generic helper with nil-row +
length-multiple-of-width validation.
- EmbeddingList search types: `entity.FloatVectorArray` + new
`Float16VectorArray` / `BFloat16VectorArray` / `BinaryVectorArray` /
`Int8VectorArray`, each dispatched to the correct
`PlaceholderType_EmbList*` in `vector2Placeholder`. Unblocks MAX_SIM /
embedding-list search against struct-array vector sub-fields from Go.
- Row-based insert helper
`(*columnBasedDataOption).WithStructArrayColumn(name,
*entity.StructSchema, []map[string]any)` that infers sub-column types
from the schema.

**Correctness fixes**

- `WithStructArrayColumn` now records construction errors on a
`deferredErr` field and surfaces them through `InsertRequest` /
`UpsertRequest` instead of panicking in the builder chain.
- `StructSchema.Validate(parent)` + `Schema.Validate()` mirror pymilvus
`StructFieldSchema._check_fields` (reject nested `ARRAY` /
`ARRAY_OF_VECTOR` / `STRUCT` sub-fields, `primary_key`, `partition_key`,
`clustering_key`, `nullable`, `auto_id`, `dynamic`, `default_value`,
nested struct schema). `Client.CreateCollection` calls `Validate()` via
interface assertion so bad schemas are rejected before the RPC.
- `columnStructArray.Len()` requires equal lengths across sub-fields and
panics loudly on drift, preventing silently malformed insert payloads.
- `columnStructArray.AppendValue` is now atomic: on sub-field error,
earlier sub-columns are rolled back to their pre-call length via
`Slice()` so the struct stays in lock-step.
- `parseVectorArrayData` validates nil `VectorField` rows, checks
`len(payload) % width == 0` and rejects `dim % 8 != 0` for binary
vectors. The 5 per-type branches are deduplicated via a generic
`splitVectorArrayRows` helper.

## Test plan

- [x] `go test -tags dynamic,test -gcflags="all=-N -l" -count=1 ./...`
in `client/` (all green): new unit tests in `entity/vectors_test.go`,
`entity/field_test.go`, `column/struct_test.go`,
`milvusclient/write_option_test.go` cover each behavior above.
- [x] End-to-end smoke test against a live Milvus server
(`chaos-testing/struct-array-test`, image
`feat-rest-v2-struct-array-20260420-0c5a964`) covering:
  - [x] `Schema.Validate()` rejects a nested-struct sub-field
- [x] `Client.CreateCollection` pre-flight rejects the same schema
before the RPC
- [x] Valid schema create / `DescribeCollection` round-trip preserves
`max_capacity` on sub-fields and correctly unwraps `ArrayOfVector` back
to `FloatVector` with `dim` preserved
- [x] `Insert` via `WithStructArrayColumn` with mixed scalar + vector
sub-fields
- [x] Builder no longer panics; missing-key deferred error surfaces on
`InsertRequest`
- [x] `Flush` / `CreateIndex(vec)` / `CreateIndex("clips[clip_emb]")`
(MAX_SIM_COSINE) / `LoadCollection`
  - [x] `Query` with `filter=id<3` returns expected rows
- [x] `Search` with `entity.FloatVectorArray` (EmbList Float
placeholder) returns hits on `anns_field="clips[clip_emb]"`
- [x] Integration test cases under `tests/go_client/testcases/`
(struct_array / struct_array_element_*) — added as new test harness;
requires a struct-array-capable server to exercise.

---------

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
zhuwenxing
2026-04-22 19:47:45 +08:00
committed by GitHub
co-authored by Claude Opus 4.7
parent 3a4e37ad90
commit d43ee11f32
22 changed files with 4654 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/column/columns.go
+143 -1
View File
@@ -174,6 +174,137 @@ func parseStructArrayData(fieldName string, structArray *schemapb.StructArrayFie
return NewColumnStructArray(fieldName, fields), nil
}
// parseVectorArrayData converts schemapb.VectorArray (per-row list of vectors) into the
// matching ColumnXxxVectorArray. Used for ArrayOfVector sub-fields of struct arrays.
func parseVectorArrayData(fieldName string, va *schemapb.VectorArray, begin, end int) (Column, error) {
rows := va.GetData()
if end < 0 || end > len(rows) {
end = len(rows)
}
if begin < 0 {
begin = 0
}
if begin > end {
begin = end
}
rows = rows[begin:end]
// VectorArray.Dim may be 0 in server search responses; fall back to inner VectorField.Dim.
dim := int(va.GetDim())
if dim == 0 {
for _, vf := range rows {
if d := int(vf.GetDim()); d > 0 {
dim = d
break
}
}
}
if dim == 0 {
return nil, fmt.Errorf("vector array %q has unknown dim", fieldName)
}
switch va.GetElementType() {
case schemapb.DataType_FloatVector:
out, err := splitVectorArrayRows(fieldName, rows, dim, func(vf *schemapb.VectorField) []float32 {
return vf.GetFloatVector().GetData()
}, func(data []float32, j, w int) []float32 {
v := make([]float32, w)
copy(v, data[j:j+w])
return v
})
if err != nil {
return nil, err
}
return NewColumnFloatVectorArray(fieldName, dim, out), nil
case schemapb.DataType_Float16Vector:
out, err := splitVectorArrayRows(fieldName, rows, dim*2, func(vf *schemapb.VectorField) []byte {
return vf.GetFloat16Vector()
}, copyByteVector)
if err != nil {
return nil, err
}
return NewColumnFloat16VectorArray(fieldName, dim, out), nil
case schemapb.DataType_BFloat16Vector:
out, err := splitVectorArrayRows(fieldName, rows, dim*2, func(vf *schemapb.VectorField) []byte {
return vf.GetBfloat16Vector()
}, copyByteVector)
if err != nil {
return nil, err
}
return NewColumnBFloat16VectorArray(fieldName, dim, out), nil
case schemapb.DataType_BinaryVector:
if dim%8 != 0 {
return nil, fmt.Errorf("binary vector array %q requires dim multiple of 8, got %d", fieldName, dim)
}
out, err := splitVectorArrayRows(fieldName, rows, dim/8, func(vf *schemapb.VectorField) []byte {
return vf.GetBinaryVector()
}, copyByteVector)
if err != nil {
return nil, err
}
return NewColumnBinaryVectorArray(fieldName, dim, out), nil
case schemapb.DataType_Int8Vector:
out, err := splitVectorArrayRows(fieldName, rows, dim, func(vf *schemapb.VectorField) []byte {
return vf.GetInt8Vector()
}, func(data []byte, j, w int) []int8 {
v := make([]int8, w)
for k := 0; k < w; k++ {
v[k] = int8(data[j+k])
}
return v
})
if err != nil {
return nil, err
}
return NewColumnInt8VectorArray(fieldName, dim, out), nil
default:
return nil, fmt.Errorf("unsupported vector array element type %s", va.GetElementType())
}
}
// splitVectorArrayRows extracts per-row flat payloads via `get` and splits each by `width` using
// `copyVector`. It validates that the flat payload length is a positive multiple of `width`, so
// that server-side protocol errors surface as clear errors instead of silent truncation or panics.
// The result is shaped as [row][vector][element].
func splitVectorArrayRows[D any, E any](
fieldName string,
rows []*schemapb.VectorField,
width int,
get func(*schemapb.VectorField) []D,
copyVector func(data []D, offset, width int) []E,
) ([][][]E, error) {
if width <= 0 {
return nil, fmt.Errorf("vector array %q has invalid row width %d", fieldName, width)
}
out := make([][][]E, 0, len(rows))
for i, vf := range rows {
if vf == nil {
return nil, fmt.Errorf("vector array %q row %d is nil", fieldName, i)
}
data := get(vf)
if len(data)%width != 0 {
return nil, fmt.Errorf("vector array %q row %d payload length %d not a multiple of row width %d",
fieldName, i, len(data), width)
}
row := make([][]E, 0, len(data)/width)
for j := 0; j+width <= len(data); j += width {
row = append(row, copyVector(data, j, width))
}
out = append(out, row)
}
return out, nil
}
func copyByteVector(data []byte, offset, width int) []byte {
v := make([]byte, width)
copy(v, data[offset:offset+width])
return v
}
func int32ToType[T ~int8 | int16](data []int32) []T {
return lo.Map(data, func(i32 int32, _ int) T {
return T(i32)
@@ -219,13 +350,24 @@ func FieldDataColumn(fd *schemapb.FieldData, begin, end int) (Column, error) {
return parseScalarData(fd.GetFieldName(), fd.GetScalars().GetStringData().GetData(), begin, end, validData, NewColumnVarChar, NewNullableColumnVarChar)
case schemapb.DataType_Array:
// handle struct array field
// handle struct array field (legacy server may use DataType_Array as top-level)
if fd.GetStructArrays() != nil {
return parseStructArrayData(fd.GetFieldName(), fd.GetStructArrays(), begin, end)
}
data := fd.GetScalars().GetArrayData()
return parseArrayData(fd.GetFieldName(), data.GetElementType(), data.GetData(), validData, begin, end)
case schemapb.DataType_ArrayOfStruct:
return parseStructArrayData(fd.GetFieldName(), fd.GetStructArrays(), begin, end)
case schemapb.DataType_ArrayOfVector:
vectors := fd.GetVectors()
va := vectors.GetVectorArray()
if va == nil {
return nil, errFieldDataTypeNotMatch
}
return parseVectorArrayData(fd.GetFieldName(), va, begin, end)
case schemapb.DataType_JSON:
return parseScalarData(fd.GetFieldName(), fd.GetScalars().GetJsonData().GetData(), begin, end, validData, NewColumnJSONBytes, NewNullableColumnJSONBytes)
client/column/struct.go
+55 -9
View File
@@ -24,6 +24,9 @@ import (
"github.com/milvus-io/milvus/client/v2/entity"
)
// columnStructArray represents a struct-array field. Each sub-column must itself be an
// "array-of-X" column (e.g. ColumnInt32Array, ColumnFloatVectorArray) so that one entry
// in the column corresponds to one row's variable-length list of struct elements.
type columnStructArray struct {
fields []Column
name string
@@ -41,14 +44,27 @@ func (c *columnStructArray) Name() string {
}
func (c *columnStructArray) Type() entity.FieldType {
// Surface as FieldTypeArray to match the user-facing schema field, whose DataType is Array
// and ElementType is Struct. This keeps processInsertColumns' type comparison happy.
return entity.FieldTypeArray
}
// Len returns the row count of the struct array. All sub-columns must have identical length;
// a mismatch indicates data corruption from a failed partial append and is reported via panic
// with a descriptive message (sub-fields are fully decoupled columns, so this check is the
// earliest opportunity to surface the invariant violation).
func (c *columnStructArray) Len() int {
for _, field := range c.fields {
return field.Len()
if len(c.fields) == 0 {
return 0
}
return 0
first := c.fields[0].Len()
for i := 1; i < len(c.fields); i++ {
if got := c.fields[i].Len(); got != first {
panic(errors.Newf("struct array %q sub-field %q length %d mismatches first sub-field %q length %d",
c.name, c.fields[i].Name(), got, c.fields[0].Name(), first).Error())
}
}
return first
}
func (c *columnStructArray) Slice(start, end int) Column {
@@ -56,13 +72,15 @@ func (c *columnStructArray) Slice(start, end int) Column {
for idx, subField := range c.fields {
fields[idx] = subField.Slice(start, end)
}
c.fields = fields
return c
return &columnStructArray{
name: c.name,
fields: fields,
}
}
func (c *columnStructArray) FieldData() *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_Array,
Type: schemapb.DataType_ArrayOfStruct,
FieldName: c.name,
Field: &schemapb.FieldData_StructArrays{
StructArrays: &schemapb.StructArrayField{
@@ -74,8 +92,36 @@ func (c *columnStructArray) FieldData() *schemapb.FieldData {
}
}
// AppendValue appends one row of struct elements. The row must be a map[string]any whose keys
// match the sub-field names; each value must match what the corresponding sub-column's AppendValue
// accepts (typically `[]T` for scalar arrays or `[][]float32`/`[][]byte` for vector arrays).
//
// If any sub-field append fails, previously appended sub-fields are rolled back to their pre-call
// lengths so sub-columns stay in lock-step.
func (c *columnStructArray) AppendValue(value any) error {
return errors.New("not implemented")
row, ok := value.(map[string]any)
if !ok {
return errors.Newf("struct array AppendValue expects map[string]any, got %T", value)
}
// pre-check all keys exist before mutating any sub-column.
for _, sub := range c.fields {
if _, present := row[sub.Name()]; !present {
return errors.Newf("struct array AppendValue: missing sub-field %q", sub.Name())
}
}
preLens := make([]int, len(c.fields))
for i, sub := range c.fields {
preLens[i] = sub.Len()
}
for i, sub := range c.fields {
if err := sub.AppendValue(row[sub.Name()]); err != nil {
for j := 0; j < i; j++ {
c.fields[j] = c.fields[j].Slice(0, preLens[j])
}
return errors.Wrapf(err, "struct array AppendValue: sub-field %q", sub.Name())
}
}
return nil
}
func (c *columnStructArray) Get(idx int) (any, error) {
@@ -107,11 +153,11 @@ func (c *columnStructArray) GetAsBool(idx int) (bool, error) {
}
func (c *columnStructArray) IsNull(idx int) (bool, error) {
return false, errors.New("not implemented")
return false, nil
}
func (c *columnStructArray) AppendNull() error {
return errors.New("not implemented")
return errors.New("struct array column does not support AppendNull")
}
func (c *columnStructArray) Nullable() bool {
client/column/struct_test.go
+193 -166
View File
@@ -34,173 +34,126 @@ type StructArraySuite struct {
func (s *StructArraySuite) TestBasic() {
name := fmt.Sprintf("struct_array_%d", rand.Intn(100))
// Create sub-fields for the struct
int32Data := []int32{1, 2, 3, 4, 5}
floatData := []float32{1.1, 2.2, 3.3, 4.4, 5.5}
varcharData := []string{"a", "b", "c", "d", "e"}
// Each row holds a variable-length array of struct elements; sub-columns are *Array types.
intRows := [][]int32{{1, 2}, {3}, {4, 5, 6}}
floatRows := [][]float32{{1.1, 2.2}, {3.3}, {4.4, 5.5, 6.6}}
strRows := [][]string{{"a", "b"}, {"c"}, {"d", "e", "f"}}
int32Col := NewColumnInt32("int_field", int32Data)
floatCol := NewColumnFloat("float_field", floatData)
varcharCol := NewColumnVarChar("varchar_field", varcharData)
int32Col := NewColumnInt32Array("int_field", intRows)
floatCol := NewColumnFloatArray("float_field", floatRows)
varcharCol := NewColumnVarCharArray("varchar_field", strRows)
fields := []Column{int32Col, floatCol, varcharCol}
column := NewColumnStructArray(name, fields)
column := NewColumnStructArray(name, []Column{int32Col, floatCol, varcharCol})
// Test Name()
s.Equal(name, column.Name())
// Test Type()
s.Equal(entity.FieldTypeArray, column.Type())
s.EqualValues(3, column.Len())
// Test Len()
s.EqualValues(5, column.Len())
// Test FieldData()
fd := column.FieldData()
s.Equal(schemapb.DataType_Array, fd.GetType())
s.Equal(schemapb.DataType_ArrayOfStruct, fd.GetType())
s.Equal(name, fd.GetFieldName())
structArrays := fd.GetStructArrays()
s.NotNil(structArrays)
s.Equal(3, len(structArrays.GetFields()))
// Verify each field in the struct array
fieldNames := []string{"int_field", "float_field", "varchar_field"}
for i, field := range structArrays.GetFields() {
s.Equal(fieldNames[i], field.GetFieldName())
// Sub-fields must be Array (not flat scalars) to match server-side schema for struct sub-fields.
for _, sub := range structArrays.GetFields() {
s.Equal(schemapb.DataType_Array, sub.GetType())
}
// Test Get() - retrieve values as map
val, err := column.Get(0)
s.NoError(err)
m, ok := val.(map[string]any)
s.True(ok)
s.Equal(int32(1), m["int_field"])
s.Equal(float32(1.1), m["float_field"])
s.Equal("a", m["varchar_field"])
s.Equal([]int32{1, 2}, m["int_field"])
s.Equal([]float32{1.1, 2.2}, m["float_field"])
s.Equal([]string{"a", "b"}, m["varchar_field"])
val, err = column.Get(2)
s.NoError(err)
m, ok = val.(map[string]any)
s.True(ok)
s.Equal(int32(3), m["int_field"])
s.Equal(float32(3.3), m["float_field"])
s.Equal("c", m["varchar_field"])
m = val.(map[string]any)
s.Equal([]int32{4, 5, 6}, m["int_field"])
}
func (s *StructArraySuite) TestVectorSubField() {
dim := 4
rows := [][][]float32{
{{0.1, 0.2, 0.3, 0.4}, {0.5, 0.6, 0.7, 0.8}},
{{1.1, 1.2, 1.3, 1.4}},
}
idRows := [][]int64{{10, 20}, {30}}
idCol := NewColumnInt64Array("id", idRows)
embCol := NewColumnFloatVectorArray("emb", dim, rows)
column := NewColumnStructArray("clips", []Column{idCol, embCol})
fd := column.FieldData()
s.Equal(schemapb.DataType_ArrayOfStruct, fd.GetType())
s.Equal(2, len(fd.GetStructArrays().GetFields()))
embFD := fd.GetStructArrays().GetFields()[1]
s.Equal(schemapb.DataType_ArrayOfVector, embFD.GetType())
va := embFD.GetVectors().GetVectorArray()
s.NotNil(va)
s.EqualValues(dim, va.GetDim())
s.Equal(schemapb.DataType_FloatVector, va.GetElementType())
s.Equal(2, len(va.GetData()))
s.EqualValues(2*dim, len(va.GetData()[0].GetFloatVector().GetData()))
s.EqualValues(1*dim, len(va.GetData()[1].GetFloatVector().GetData()))
}
func (s *StructArraySuite) TestSlice() {
name := "struct_array_slice"
intRows := [][]int64{{10}, {20, 21}, {30, 31, 32}, {40}, {50, 51}}
boolRows := [][]bool{{true}, {false, true}, {true, false, true}, {false}, {true, false}}
// Create sub-fields
int64Data := []int64{10, 20, 30, 40, 50}
boolData := []bool{true, false, true, false, true}
int64Col := NewColumnInt64Array("id", intRows)
boolCol := NewColumnBoolArray("flag", boolRows)
column := NewColumnStructArray("struct_array_slice", []Column{int64Col, boolCol})
int64Col := NewColumnInt64("id", int64Data)
boolCol := NewColumnBool("flag", boolData)
fields := []Column{int64Col, boolCol}
column := NewColumnStructArray(name, fields)
// Test Slice(1, 4) - should slice all sub-fields
sliced := column.Slice(1, 4)
s.NotNil(sliced)
s.EqualValues(3, sliced.Len())
// Verify sliced data
val, err := sliced.Get(0)
s.NoError(err)
m, ok := val.(map[string]any)
s.True(ok)
s.Equal(int64(20), m["id"])
s.Equal(false, m["flag"])
val, err = sliced.Get(2)
s.NoError(err)
m, ok = val.(map[string]any)
s.True(ok)
s.Equal(int64(40), m["id"])
s.Equal(false, m["flag"])
m := val.(map[string]any)
s.Equal([]int64{20, 21}, m["id"])
s.Equal([]bool{false, true}, m["flag"])
}
func (s *StructArraySuite) TestNullable() {
name := "struct_array_nullable"
func (s *StructArraySuite) TestAppendValue() {
intCol := NewColumnInt32Array("a", nil)
strCol := NewColumnVarCharArray("b", nil)
column := NewColumnStructArray("rows", []Column{intCol, strCol})
// Create sub-fields
int32Data := []int32{1, 2, 3}
int32Col := NewColumnInt32("field1", int32Data)
s.NoError(column.AppendValue(map[string]any{"a": []int32{1, 2}, "b": []string{"x", "y"}}))
s.NoError(column.AppendValue(map[string]any{"a": []int32{3}, "b": []string{"z"}}))
s.EqualValues(2, column.Len())
varcharData := []string{"x", "y", "z"}
varcharCol := NewColumnVarChar("field2", varcharData)
fields := []Column{int32Col, varcharCol}
column := NewColumnStructArray(name, fields)
// Test Nullable() - should return false by default
s.False(column.Nullable())
// Test ValidateNullable() - should validate all sub-fields
err := column.ValidateNullable()
s.NoError(err)
// Test CompactNullableValues() - should not panic
s.NotPanics(func() {
column.CompactNullableValues()
})
}
func (s *StructArraySuite) TestNotImplementedMethods() {
name := "struct_array_not_impl"
int32Data := []int32{1, 2, 3}
int32Col := NewColumnInt32("field1", int32Data)
fields := []Column{int32Col}
column := NewColumnStructArray(name, fields)
// Test AppendValue - should return error
err := column.AppendValue(map[string]any{"field1": int32(4)})
// missing sub-field
err := column.AppendValue(map[string]any{"a": []int32{4}})
s.Error(err)
s.Contains(err.Error(), "not implemented")
// Test GetAsInt64 - should return error
_, err = column.GetAsInt64(0)
// wrong shape (scalar instead of array)
err = column.AppendValue(map[string]any{"a": int32(1), "b": []string{"q"}})
s.Error(err)
s.Contains(err.Error(), "not implemented")
// Test GetAsString - should return error
_, err = column.GetAsString(0)
s.Error(err)
s.Contains(err.Error(), "not implemented")
// Test GetAsDouble - should return error
_, err = column.GetAsDouble(0)
s.Error(err)
s.Contains(err.Error(), "not implemented")
// Test GetAsBool - should return error
_, err = column.GetAsBool(0)
s.Error(err)
s.Contains(err.Error(), "not implemented")
// Test IsNull - should return error
_, err = column.IsNull(0)
s.Error(err)
s.Contains(err.Error(), "not implemented")
// Test AppendNull - should return error
err = column.AppendNull()
s.Error(err)
s.Contains(err.Error(), "not implemented")
}
func (s *StructArraySuite) TestParseStructArrayData() {
// Create a FieldData with struct array
int32FieldData := &schemapb.FieldData{
Type: schemapb.DataType_Int32,
Type: schemapb.DataType_Array,
FieldName: "age",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{10, 20, 30},
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
ElementType: schemapb.DataType_Int32,
Data: []*schemapb.ScalarField{
{Data: &schemapb.ScalarField_IntData{IntData: &schemapb.IntArray{Data: []int32{10, 11}}}},
{Data: &schemapb.ScalarField_IntData{IntData: &schemapb.IntArray{Data: []int32{20}}}},
{Data: &schemapb.ScalarField_IntData{IntData: &schemapb.IntArray{Data: []int32{30, 31, 32}}}},
},
},
},
},
@@ -208,13 +161,18 @@ func (s *StructArraySuite) TestParseStructArrayData() {
}
varcharFieldData := &schemapb.FieldData{
Type: schemapb.DataType_VarChar,
Type: schemapb.DataType_Array,
FieldName: "name",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: []string{"alice", "bob", "charlie"},
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
ElementType: schemapb.DataType_VarChar,
Data: []*schemapb.ScalarField{
{Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"alice", "ann"}}}},
{Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"bob"}}}},
{Data: &schemapb.ScalarField_StringData{StringData: &schemapb.StringArray{Data: []string{"c1", "c2", "c3"}}}},
},
},
},
},
@@ -225,66 +183,135 @@ func (s *StructArraySuite) TestParseStructArrayData() {
Fields: []*schemapb.FieldData{int32FieldData, varcharFieldData},
}
// Test parseStructArrayData
column, err := parseStructArrayData("person", structArrayField, 0, -1)
col, err := parseStructArrayData("person", structArrayField, 0, -1)
s.NoError(err)
s.NotNil(column)
s.Equal("person", column.Name())
s.Equal(entity.FieldTypeArray, column.Type())
s.NotNil(col)
s.Equal("person", col.Name())
s.Equal(entity.FieldTypeArray, col.Type())
// Verify we can get values
val, err := column.Get(0)
val, err := col.Get(0)
s.NoError(err)
m, ok := val.(map[string]any)
s.True(ok)
s.Equal(int32(10), m["age"])
s.Equal("alice", m["name"])
m := val.(map[string]any)
s.Equal([]int32{10, 11}, m["age"])
s.Equal([]string{"alice", "ann"}, m["name"])
val, err = column.Get(1)
val, err = col.Get(1)
s.NoError(err)
m, ok = val.(map[string]any)
s.True(ok)
s.Equal(int32(20), m["age"])
s.Equal("bob", m["name"])
m = val.(map[string]any)
s.Equal([]int32{20}, m["age"])
s.Equal([]string{"bob"}, m["name"])
}
func (s *StructArraySuite) TestParseStructArrayDataWithRange() {
// Create a FieldData with struct array
int64FieldData := &schemapb.FieldData{
Type: schemapb.DataType_Int64,
FieldName: "id",
func (s *StructArraySuite) TestParseTopLevelArrayOfStruct() {
// Verify FieldDataColumn dispatches DataType_ArrayOfStruct to parseStructArrayData.
int32Sub := &schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: "x",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: []int64{100, 200, 300, 400, 500},
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
ElementType: schemapb.DataType_Int32,
Data: []*schemapb.ScalarField{
{Data: &schemapb.ScalarField_IntData{IntData: &schemapb.IntArray{Data: []int32{1, 2}}}},
},
},
},
},
},
}
top := &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfStruct,
FieldName: "wrap",
Field: &schemapb.FieldData_StructArrays{
StructArrays: &schemapb.StructArrayField{Fields: []*schemapb.FieldData{int32Sub}},
},
}
col, err := FieldDataColumn(top, 0, -1)
s.NoError(err)
s.Equal("wrap", col.Name())
val, err := col.Get(0)
s.NoError(err)
s.Equal([]int32{1, 2}, val.(map[string]any)["x"])
}
structArrayField := &schemapb.StructArrayField{
Fields: []*schemapb.FieldData{int64FieldData},
func (s *StructArraySuite) TestParseVectorArrayDataErrors() {
mkFD := func(elemType schemapb.DataType, dim int64, rows []*schemapb.VectorField) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfVector,
FieldName: "emb",
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_VectorArray{
VectorArray: &schemapb.VectorArray{
Dim: dim, ElementType: elemType, Data: rows,
},
},
},
},
}
}
// Test parseStructArrayData with range [1, 4)
column, err := parseStructArrayData("data", structArrayField, 1, 4)
s.NoError(err)
s.NotNil(column)
s.Run("unknown dim rejected", func() {
fd := mkFD(schemapb.DataType_FloatVector, 0, nil)
_, err := FieldDataColumn(fd, 0, -1)
s.Error(err)
})
// Verify sliced data (should contain indices 1, 2, 3)
val, err := column.Get(0)
s.NoError(err)
m, ok := val.(map[string]any)
s.True(ok)
s.Equal(int64(200), m["id"])
s.Run("payload not a multiple of dim", func() {
// dim=4 but row has 5 floats -> must error, not silently truncate.
row := &schemapb.VectorField{
Dim: 4,
Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4, 5}}},
}
fd := mkFD(schemapb.DataType_FloatVector, 4, []*schemapb.VectorField{row})
_, err := FieldDataColumn(fd, 0, -1)
s.Error(err)
})
val, err = column.Get(2)
s.NoError(err)
m, ok = val.(map[string]any)
s.True(ok)
s.Equal(int64(400), m["id"])
s.Run("binary dim not multiple of 8", func() {
row := &schemapb.VectorField{
Dim: 4,
Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0}},
}
fd := mkFD(schemapb.DataType_BinaryVector, 4, []*schemapb.VectorField{row})
_, err := FieldDataColumn(fd, 0, -1)
s.Error(err)
})
s.Run("nil row rejected", func() {
fd := mkFD(schemapb.DataType_FloatVector, 4, []*schemapb.VectorField{nil})
_, err := FieldDataColumn(fd, 0, -1)
s.Error(err)
})
}
func (s *StructArraySuite) TestAppendValueRollback() {
intCol := NewColumnInt32Array("a", nil)
strCol := NewColumnVarCharArray("b", nil)
col := NewColumnStructArray("rows", []Column{intCol, strCol}).(*columnStructArray)
// Seed with one good row so both sub-columns are at length 1.
s.NoError(col.AppendValue(map[string]any{"a": []int32{1}, "b": []string{"x"}}))
s.EqualValues(1, col.Len())
// Second row: sub-field "a" accepts the []int32, but "b" gets wrong type —
// rollback must restore sub-column "a" to length 1 so the struct stays in lock-step.
err := col.AppendValue(map[string]any{"a": []int32{2}, "b": 42})
s.Error(err)
s.EqualValues(1, col.fields[0].Len(), "sub-field 'a' must be rolled back")
s.EqualValues(1, col.fields[1].Len(), "sub-field 'b' must not have been appended")
s.EqualValues(1, col.Len(), "struct array length stays consistent after rollback")
}
func (s *StructArraySuite) TestLenMismatchPanics() {
// Manually drift sub-column lengths to simulate a prior corruption and verify Len reports it.
intCol := NewColumnInt32Array("a", [][]int32{{1}, {2}})
strCol := NewColumnVarCharArray("b", [][]string{{"x"}})
col := NewColumnStructArray("rows", []Column{intCol, strCol})
s.Panics(func() { _ = col.Len() })
}
func TestStructArray(t *testing.T) {
client/column/vector_array.go
+335
View File
@@ -0,0 +1,335 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
// columnVectorArrayBase implements `Column` for vector-array sub-fields of struct array.
// Each row contains a variable-length list of vectors of equal `dim`.
type columnVectorArrayBase[T entity.Vector] struct {
name string
fieldType entity.FieldType // e.g. FieldTypeArray (top-level type for column matching)
elementType entity.FieldType // underlying vector type, e.g. FieldTypeFloatVector
dim int
values [][]T // values[i] = list of vectors for row i
}
func (c *columnVectorArrayBase[T]) Name() string {
return c.name
}
func (c *columnVectorArrayBase[T]) Type() entity.FieldType {
return c.fieldType
}
func (c *columnVectorArrayBase[T]) ElementType() entity.FieldType {
return c.elementType
}
func (c *columnVectorArrayBase[T]) Dim() int {
return c.dim
}
func (c *columnVectorArrayBase[T]) Len() int {
return len(c.values)
}
func (c *columnVectorArrayBase[T]) Get(idx int) (any, error) {
if idx < 0 || idx >= len(c.values) {
return nil, errors.Newf("index %d out of range[0, %d)", idx, len(c.values))
}
return c.values[idx], nil
}
func (c *columnVectorArrayBase[T]) AppendValue(value any) error {
v, ok := value.([]T)
if !ok {
return errors.Newf("unexpected append value type %T, field type %v", value, c.fieldType)
}
c.values = append(c.values, v)
return nil
}
func (c *columnVectorArrayBase[T]) GetAsInt64(_ int) (int64, error) {
return 0, errors.New("vector array column does not support GetAsInt64")
}
func (c *columnVectorArrayBase[T]) GetAsString(_ int) (string, error) {
return "", errors.New("vector array column does not support GetAsString")
}
func (c *columnVectorArrayBase[T]) GetAsDouble(_ int) (float64, error) {
return 0, errors.New("vector array column does not support GetAsDouble")
}
func (c *columnVectorArrayBase[T]) GetAsBool(_ int) (bool, error) {
return false, errors.New("vector array column does not support GetAsBool")
}
func (c *columnVectorArrayBase[T]) IsNull(_ int) (bool, error) {
return false, nil
}
func (c *columnVectorArrayBase[T]) AppendNull() error {
return errors.New("vector array column does not support AppendNull")
}
func (c *columnVectorArrayBase[T]) Nullable() bool { return false }
func (c *columnVectorArrayBase[T]) SetNullable(_ bool) {}
func (c *columnVectorArrayBase[T]) ValidateNullable() error { return nil }
func (c *columnVectorArrayBase[T]) CompactNullableValues() {}
func (c *columnVectorArrayBase[T]) ValidCount() int { return c.Len() }
func (c *columnVectorArrayBase[T]) Slice(start, end int) Column {
if end == -1 || end > len(c.values) {
end = len(c.values)
}
if start > end {
start = end
}
return &columnVectorArrayBase[T]{
name: c.name,
fieldType: c.fieldType,
elementType: c.elementType,
dim: c.dim,
values: c.values[start:end],
}
}
func (c *columnVectorArrayBase[T]) FieldData() *schemapb.FieldData {
rows := make([]*schemapb.VectorField, 0, len(c.values))
for _, row := range c.values {
rows = append(rows, values2Vectors(row, c.elementType, int64(c.dim)))
}
return &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfVector,
FieldName: c.name,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(c.dim),
Data: &schemapb.VectorField_VectorArray{
VectorArray: &schemapb.VectorArray{
Dim: int64(c.dim),
Data: rows,
ElementType: schemapb.DataType(c.elementType),
},
},
},
},
}
}
/* float vector array */
type ColumnFloatVectorArray struct {
*columnVectorArrayBase[entity.FloatVector]
}
func NewColumnFloatVectorArray(fieldName string, dim int, data [][][]float32) *ColumnFloatVectorArray {
values := make([][]entity.FloatVector, 0, len(data))
for _, row := range data {
vrow := make([]entity.FloatVector, 0, len(row))
for _, v := range row {
vrow = append(vrow, entity.FloatVector(v))
}
values = append(values, vrow)
}
return &ColumnFloatVectorArray{
columnVectorArrayBase: &columnVectorArrayBase[entity.FloatVector]{
name: fieldName,
fieldType: entity.FieldTypeArray,
elementType: entity.FieldTypeFloatVector,
dim: dim,
values: values,
},
}
}
// AppendValue accepts `[]entity.FloatVector` or `[][]float32` for one row.
func (c *ColumnFloatVectorArray) AppendValue(value any) error {
switch v := value.(type) {
case []entity.FloatVector:
c.values = append(c.values, v)
case [][]float32:
row := make([]entity.FloatVector, 0, len(v))
for _, x := range v {
row = append(row, entity.FloatVector(x))
}
c.values = append(c.values, row)
default:
return errors.Newf("unexpected append value type %T, field type %v", value, c.elementType)
}
return nil
}
func appendByteVectorArrayRow[T ~[]byte](values *[][]T, value any) error {
switch v := value.(type) {
case []T:
*values = append(*values, v)
case [][]byte:
row := make([]T, 0, len(v))
for _, x := range v {
row = append(row, T(x))
}
*values = append(*values, row)
default:
return errors.Newf("unexpected append value type %T", value)
}
return nil
}
/* float16 vector array */
type ColumnFloat16VectorArray struct {
*columnVectorArrayBase[entity.Float16Vector]
}
func (c *ColumnFloat16VectorArray) AppendValue(value any) error {
return appendByteVectorArrayRow(&c.values, value)
}
func NewColumnFloat16VectorArray(fieldName string, dim int, data [][][]byte) *ColumnFloat16VectorArray {
values := make([][]entity.Float16Vector, 0, len(data))
for _, row := range data {
vrow := make([]entity.Float16Vector, 0, len(row))
for _, v := range row {
vrow = append(vrow, entity.Float16Vector(v))
}
values = append(values, vrow)
}
return &ColumnFloat16VectorArray{
columnVectorArrayBase: &columnVectorArrayBase[entity.Float16Vector]{
name: fieldName,
fieldType: entity.FieldTypeArray,
elementType: entity.FieldTypeFloat16Vector,
dim: dim,
values: values,
},
}
}
/* bfloat16 vector array */
type ColumnBFloat16VectorArray struct {
*columnVectorArrayBase[entity.BFloat16Vector]
}
func (c *ColumnBFloat16VectorArray) AppendValue(value any) error {
return appendByteVectorArrayRow(&c.values, value)
}
func NewColumnBFloat16VectorArray(fieldName string, dim int, data [][][]byte) *ColumnBFloat16VectorArray {
values := make([][]entity.BFloat16Vector, 0, len(data))
for _, row := range data {
vrow := make([]entity.BFloat16Vector, 0, len(row))
for _, v := range row {
vrow = append(vrow, entity.BFloat16Vector(v))
}
values = append(values, vrow)
}
return &ColumnBFloat16VectorArray{
columnVectorArrayBase: &columnVectorArrayBase[entity.BFloat16Vector]{
name: fieldName,
fieldType: entity.FieldTypeArray,
elementType: entity.FieldTypeBFloat16Vector,
dim: dim,
values: values,
},
}
}
/* binary vector array */
type ColumnBinaryVectorArray struct {
*columnVectorArrayBase[entity.BinaryVector]
}
func (c *ColumnBinaryVectorArray) AppendValue(value any) error {
return appendByteVectorArrayRow(&c.values, value)
}
func NewColumnBinaryVectorArray(fieldName string, dim int, data [][][]byte) *ColumnBinaryVectorArray {
values := make([][]entity.BinaryVector, 0, len(data))
for _, row := range data {
vrow := make([]entity.BinaryVector, 0, len(row))
for _, v := range row {
vrow = append(vrow, entity.BinaryVector(v))
}
values = append(values, vrow)
}
return &ColumnBinaryVectorArray{
columnVectorArrayBase: &columnVectorArrayBase[entity.BinaryVector]{
name: fieldName,
fieldType: entity.FieldTypeArray,
elementType: entity.FieldTypeBinaryVector,
dim: dim,
values: values,
},
}
}
/* int8 vector array */
type ColumnInt8VectorArray struct {
*columnVectorArrayBase[entity.Int8Vector]
}
// AppendValue accepts `[]entity.Int8Vector` or `[][]int8` for one row.
func (c *ColumnInt8VectorArray) AppendValue(value any) error {
switch v := value.(type) {
case []entity.Int8Vector:
c.values = append(c.values, v)
case [][]int8:
row := make([]entity.Int8Vector, 0, len(v))
for _, x := range v {
row = append(row, entity.Int8Vector(x))
}
c.values = append(c.values, row)
default:
return errors.Newf("unexpected append value type %T, field type %v", value, c.elementType)
}
return nil
}
func NewColumnInt8VectorArray(fieldName string, dim int, data [][][]int8) *ColumnInt8VectorArray {
values := make([][]entity.Int8Vector, 0, len(data))
for _, row := range data {
vrow := make([]entity.Int8Vector, 0, len(row))
for _, v := range row {
vrow = append(vrow, entity.Int8Vector(v))
}
values = append(values, vrow)
}
return &ColumnInt8VectorArray{
columnVectorArrayBase: &columnVectorArrayBase[entity.Int8Vector]{
name: fieldName,
fieldType: entity.FieldTypeArray,
elementType: entity.FieldTypeInt8Vector,
dim: dim,
values: values,
},
}
}
client/column/vector_array_test.go
+387
View File
@@ -0,0 +1,387 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package column
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/entity"
)
type VectorArraySuite struct {
suite.Suite
}
func (s *VectorArraySuite) TestFloatVectorArrayBasic() {
dim := 4
rows := [][][]float32{
{{0.1, 0.2, 0.3, 0.4}, {0.5, 0.6, 0.7, 0.8}},
{{1.1, 1.2, 1.3, 1.4}},
}
col := NewColumnFloatVectorArray("emb", dim, rows)
s.Equal("emb", col.Name())
s.Equal(entity.FieldTypeArray, col.Type())
s.Equal(entity.FieldTypeFloatVector, col.ElementType())
s.Equal(dim, col.Dim())
s.Equal(2, col.Len())
s.Equal(2, col.ValidCount())
s.False(col.Nullable())
v, err := col.Get(0)
s.NoError(err)
row0, ok := v.([]entity.FloatVector)
s.Require().True(ok)
s.Equal(2, len(row0))
// Out-of-range Get.
_, err = col.Get(-1)
s.Error(err)
_, err = col.Get(99)
s.Error(err)
// Scalar conversions are unsupported on vector array columns.
_, err = col.GetAsInt64(0)
s.Error(err)
_, err = col.GetAsString(0)
s.Error(err)
_, err = col.GetAsDouble(0)
s.Error(err)
_, err = col.GetAsBool(0)
s.Error(err)
// Nullable helpers are no-ops / zero-valued for vector arrays.
isNull, err := col.IsNull(0)
s.NoError(err)
s.False(isNull)
s.Error(col.AppendNull())
col.SetNullable(true)
s.False(col.Nullable())
s.NoError(col.ValidateNullable())
col.CompactNullableValues()
// AppendValue via both canonical shapes.
s.NoError(col.AppendValue([]entity.FloatVector{entity.FloatVector([]float32{2.1, 2.2, 2.3, 2.4})}))
s.NoError(col.AppendValue([][]float32{{3.1, 3.2, 3.3, 3.4}}))
s.Equal(4, col.Len())
// AppendValue rejects bad shapes.
s.Error(col.AppendValue([]int{1, 2, 3}))
// FieldData round-trip.
fd := col.FieldData()
s.Equal(schemapb.DataType_ArrayOfVector, fd.GetType())
s.Equal("emb", fd.GetFieldName())
va := fd.GetVectors().GetVectorArray()
s.Require().NotNil(va)
s.EqualValues(dim, va.GetDim())
s.Equal(schemapb.DataType_FloatVector, va.GetElementType())
s.Equal(4, len(va.GetData()))
}
func (s *VectorArraySuite) TestFloat16VectorArrayBasic() {
dim := 4
byteRow := make([]byte, dim*2)
rows := [][][]byte{{byteRow, byteRow}, {byteRow}}
col := NewColumnFloat16VectorArray("emb", dim, rows)
s.Equal(entity.FieldTypeFloat16Vector, col.ElementType())
s.Equal(2, col.Len())
// AppendValue variants.
s.NoError(col.AppendValue([]entity.Float16Vector{entity.Float16Vector(byteRow)}))
s.NoError(col.AppendValue([][]byte{byteRow}))
s.Error(col.AppendValue(123))
s.Equal(4, col.Len())
fd := col.FieldData()
s.Equal(schemapb.DataType_ArrayOfVector, fd.GetType())
s.Equal(schemapb.DataType_Float16Vector, fd.GetVectors().GetVectorArray().GetElementType())
}
func (s *VectorArraySuite) TestBFloat16VectorArrayBasic() {
dim := 4
byteRow := make([]byte, dim*2)
rows := [][][]byte{{byteRow}}
col := NewColumnBFloat16VectorArray("emb", dim, rows)
s.Equal(entity.FieldTypeBFloat16Vector, col.ElementType())
s.Equal(1, col.Len())
s.NoError(col.AppendValue([]entity.BFloat16Vector{entity.BFloat16Vector(byteRow)}))
s.NoError(col.AppendValue([][]byte{byteRow}))
s.Error(col.AppendValue("bad"))
s.Equal(3, col.Len())
fd := col.FieldData()
s.Equal(schemapb.DataType_BFloat16Vector, fd.GetVectors().GetVectorArray().GetElementType())
}
func (s *VectorArraySuite) TestBinaryVectorArrayBasic() {
dim := 8 // binary dim is bits; 1 byte per vector.
byteRow := make([]byte, dim/8)
rows := [][][]byte{{byteRow, byteRow}}
col := NewColumnBinaryVectorArray("emb", dim, rows)
s.Equal(entity.FieldTypeBinaryVector, col.ElementType())
s.Equal(1, col.Len())
s.NoError(col.AppendValue([]entity.BinaryVector{entity.BinaryVector(byteRow)}))
s.NoError(col.AppendValue([][]byte{byteRow}))
s.Error(col.AppendValue(42))
s.Equal(3, col.Len())
fd := col.FieldData()
s.Equal(schemapb.DataType_BinaryVector, fd.GetVectors().GetVectorArray().GetElementType())
}
func (s *VectorArraySuite) TestInt8VectorArrayBasic() {
dim := 4
rows := [][][]int8{{{1, 2, 3, 4}, {5, 6, 7, 8}}, {{9, 10, 11, 12}}}
col := NewColumnInt8VectorArray("emb", dim, rows)
s.Equal(entity.FieldTypeInt8Vector, col.ElementType())
s.Equal(2, col.Len())
s.NoError(col.AppendValue([]entity.Int8Vector{entity.Int8Vector([]int8{1, 2, 3, 4})}))
s.NoError(col.AppendValue([][]int8{{5, 6, 7, 8}}))
s.Error(col.AppendValue("bad"))
s.Equal(4, col.Len())
fd := col.FieldData()
s.Equal(schemapb.DataType_Int8Vector, fd.GetVectors().GetVectorArray().GetElementType())
}
func (s *VectorArraySuite) TestBaseAppendValueRejectsWrongType() {
dim := 4
col := NewColumnFloat16VectorArray("emb", dim, nil)
// The columnVectorArrayBase AppendValue path (via embedded struct) rejects mismatched types.
s.Error(col.columnVectorArrayBase.AppendValue([]int{1, 2}))
// The positive path is exercised in TestFloat16VectorArrayBasic.
}
func (s *VectorArraySuite) TestParseVectorArrayDataFloatSuccess() {
dim := 4
row := &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4, 5, 6, 7, 8}},
},
}
fd := &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfVector,
FieldName: "emb",
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_VectorArray{
VectorArray: &schemapb.VectorArray{
Dim: int64(dim),
ElementType: schemapb.DataType_FloatVector,
Data: []*schemapb.VectorField{row, row},
},
},
},
},
}
col, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
s.Equal(2, col.Len())
fv, ok := col.(*ColumnFloatVectorArray)
s.Require().True(ok)
s.Equal(dim, fv.Dim())
}
func (s *VectorArraySuite) TestParseVectorArrayDataByteTypes() {
dim := 4
// Build a single payload holding 2 inner vectors of `dim*2` bytes each.
fp16Row := &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_Float16Vector{Float16Vector: make([]byte, dim*2*2)},
}
bf16Row := &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: make([]byte, dim*2*2)},
}
int8Row := &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_Int8Vector{Int8Vector: make([]byte, dim*2)},
}
// Binary: dim=16 bits -> 2 bytes per inner vector.
binRow := &schemapb.VectorField{
Dim: 16,
Data: &schemapb.VectorField_BinaryVector{BinaryVector: make([]byte, 2*2)},
}
cases := []struct {
name string
elem schemapb.DataType
innerFD *schemapb.VectorField
arrDim int64
wantCol Column
}{
{"float16", schemapb.DataType_Float16Vector, fp16Row, int64(dim), (*ColumnFloat16VectorArray)(nil)},
{"bfloat16", schemapb.DataType_BFloat16Vector, bf16Row, int64(dim), (*ColumnBFloat16VectorArray)(nil)},
{"int8", schemapb.DataType_Int8Vector, int8Row, int64(dim), (*ColumnInt8VectorArray)(nil)},
{"binary", schemapb.DataType_BinaryVector, binRow, 16, (*ColumnBinaryVectorArray)(nil)},
}
for _, c := range cases {
s.Run(c.name, func() {
fd := &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfVector,
FieldName: "emb",
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: c.arrDim,
Data: &schemapb.VectorField_VectorArray{
VectorArray: &schemapb.VectorArray{
Dim: c.arrDim,
ElementType: c.elem,
Data: []*schemapb.VectorField{c.innerFD},
},
},
},
},
}
col, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
s.Equal(1, col.Len())
s.IsType(c.wantCol, col)
})
}
}
func (s *VectorArraySuite) TestParseVectorArrayDataUnsupportedElement() {
fd := &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfVector,
FieldName: "emb",
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 4,
Data: &schemapb.VectorField_VectorArray{
VectorArray: &schemapb.VectorArray{
Dim: 4,
ElementType: schemapb.DataType_SparseFloatVector, // not supported inside ArrayOfVector
Data: nil,
},
},
},
},
}
_, err := FieldDataColumn(fd, 0, -1)
s.Error(err)
}
func (s *VectorArraySuite) TestParseVectorArrayDataBeginEndClamping() {
dim := 2
row := &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{Data: []float32{1, 2}},
},
}
fd := &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfVector,
FieldName: "emb",
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_VectorArray{
VectorArray: &schemapb.VectorArray{
Dim: int64(dim),
ElementType: schemapb.DataType_FloatVector,
Data: []*schemapb.VectorField{row, row, row},
},
},
},
},
}
// negative begin gets clamped to 0; end > len clamped to len.
col, err := FieldDataColumn(fd, -5, 10)
s.NoError(err)
s.Equal(3, col.Len())
// begin > end collapses to empty.
col2, err := FieldDataColumn(fd, 10, 1)
s.NoError(err)
s.Equal(0, col2.Len())
}
func (s *VectorArraySuite) TestParseVectorArrayDataFallbackDim() {
// Outer Dim=0 should fall back to the first non-zero inner VectorField.Dim.
row := &schemapb.VectorField{
Dim: 4,
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}},
},
}
fd := &schemapb.FieldData{
Type: schemapb.DataType_ArrayOfVector,
FieldName: "emb",
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 0,
Data: &schemapb.VectorField_VectorArray{
VectorArray: &schemapb.VectorArray{
Dim: 0,
ElementType: schemapb.DataType_FloatVector,
Data: []*schemapb.VectorField{row},
},
},
},
},
}
col, err := FieldDataColumn(fd, 0, -1)
s.NoError(err)
fv, ok := col.(*ColumnFloatVectorArray)
s.Require().True(ok)
s.Equal(4, fv.Dim())
}
func (s *VectorArraySuite) TestSlice() {
dim := 2
rows := [][][]float32{
{{1, 2}},
{{3, 4}, {5, 6}},
{{7, 8}},
{{9, 10}, {11, 12}},
}
col := NewColumnFloatVectorArray("emb", dim, rows)
sliced := col.Slice(1, 3)
s.Equal(2, sliced.Len())
// end clamped to len.
sliced2 := col.Slice(2, 99)
s.Equal(2, sliced2.Len())
// end == -1 means to the end.
sliced3 := col.Slice(1, -1)
s.Equal(3, sliced3.Len())
// start > end is clamped to end.
sliced4 := col.Slice(3, 1)
s.Equal(0, sliced4.Len())
}
func TestVectorArray(t *testing.T) {
suite.Run(t, new(VectorArraySuite))
}
client/entity/field.go
+57
View File
@@ -502,3 +502,60 @@ func (f *StructSchema) WithField(field *Field) *StructSchema {
f.Fields = append(f.Fields, field)
return f
}
// Validate checks struct-array sub-field rules aligned with pymilvus StructFieldSchema._check_fields.
// A sub-field MUST NOT be Array / Struct itself, and MUST NOT carry top-level-only flags such as
// primary/partition/clustering key, nullable, autoID, default value, or be dynamic.
// `parentName` is used to contextualize the error message.
func (f *StructSchema) Validate(parentName string) error {
if f == nil {
return errors.New("nil struct schema")
}
if len(f.Fields) == 0 {
return errors.Newf("struct array field %q must have at least one sub-field", parentName)
}
seen := make(map[string]struct{}, len(f.Fields))
for _, sub := range f.Fields {
if sub == nil {
return errors.Newf("struct array field %q contains nil sub-field", parentName)
}
if sub.Name == "" {
return errors.Newf("struct array field %q contains sub-field with empty name", parentName)
}
if _, dup := seen[sub.Name]; dup {
return errors.Newf("struct array field %q contains duplicate sub-field %q", parentName, sub.Name)
}
seen[sub.Name] = struct{}{}
switch sub.DataType {
case FieldTypeArray, FieldTypeStruct:
return errors.Newf("struct array field %q sub-field %q: nested array/struct sub-field is not allowed", parentName, sub.Name)
case FieldTypeSparseVector:
return errors.Newf("struct array field %q sub-field %q: sparse vector sub-field is not supported", parentName, sub.Name)
}
if sub.PrimaryKey {
return errors.Newf("struct array field %q sub-field %q must not be primary key", parentName, sub.Name)
}
if sub.AutoID {
return errors.Newf("struct array field %q sub-field %q must not be auto id", parentName, sub.Name)
}
if sub.IsPartitionKey {
return errors.Newf("struct array field %q sub-field %q must not be partition key", parentName, sub.Name)
}
if sub.IsClusteringKey {
return errors.Newf("struct array field %q sub-field %q must not be clustering key", parentName, sub.Name)
}
if sub.IsDynamic {
return errors.Newf("struct array field %q sub-field %q must not be dynamic", parentName, sub.Name)
}
if sub.Nullable {
return errors.Newf("struct array field %q sub-field %q must not be nullable", parentName, sub.Name)
}
if sub.DefaultValue != nil {
return errors.Newf("struct array field %q sub-field %q must not carry default value", parentName, sub.Name)
}
if sub.StructSchema != nil {
return errors.Newf("struct array field %q sub-field %q must not nest a struct schema", parentName, sub.Name)
}
}
return nil
}
client/entity/field_test.go
+138
View File
@@ -94,6 +94,144 @@ func TestStructSchema(t *testing.T) {
assert.Equal(t, FieldTypeFloat, schema.Fields[2].DataType)
}
func TestStructSchemaValidate(t *testing.T) {
t.Run("ok: mixed scalar and vector sub-fields", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("tag").WithDataType(FieldTypeVarChar).WithMaxLength(64)).
WithField(NewField().WithName("emb").WithDataType(FieldTypeFloatVector).WithDim(8))
assert.NoError(t, ss.Validate("clips"))
})
t.Run("empty sub-fields", func(t *testing.T) {
assert.Error(t, NewStructSchema().Validate("clips"))
})
t.Run("duplicate sub-field name", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt32)).
WithField(NewField().WithName("a").WithDataType(FieldTypeInt32))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject nested array", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeArray).WithElementType(FieldTypeInt32))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject primary key / nullable / autoID", func(t *testing.T) {
assert.Error(t, NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt32).WithIsPrimaryKey(true)).
Validate("clips"))
assert.Error(t, NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt32).WithNullable(true)).
Validate("clips"))
assert.Error(t, NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt32).WithIsAutoID(true)).
Validate("clips"))
})
t.Run("nil struct schema", func(t *testing.T) {
var ss *StructSchema
assert.Error(t, ss.Validate("clips"))
})
t.Run("nil sub-field", func(t *testing.T) {
ss := &StructSchema{Fields: []*Field{nil}}
assert.Error(t, ss.Validate("clips"))
})
t.Run("sub-field with empty name", func(t *testing.T) {
ss := NewStructSchema().WithField(NewField().WithDataType(FieldTypeInt32))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject nested struct sub-field", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeStruct))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject sparse vector sub-field", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeSparseVector))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject partition key sub-field", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt64).WithIsPartitionKey(true))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject clustering key sub-field", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt64).WithIsClusteringKey(true))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject dynamic sub-field", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt64).WithIsDynamic(true))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject default value sub-field", func(t *testing.T) {
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt64).WithDefaultValueLong(7))
assert.Error(t, ss.Validate("clips"))
})
t.Run("reject nested struct schema inside sub-field", func(t *testing.T) {
inner := NewStructSchema().WithField(NewField().WithName("x").WithDataType(FieldTypeInt32))
ss := NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeInt64).WithStructSchema(inner))
assert.Error(t, ss.Validate("clips"))
})
}
func TestSchemaValidateExtra(t *testing.T) {
t.Run("nil schema", func(t *testing.T) {
var s *Schema
assert.Error(t, s.Validate())
})
t.Run("schema with nil field", func(t *testing.T) {
s := &Schema{Fields: []*Field{nil}}
assert.Error(t, s.Validate())
})
t.Run("non-struct fields are skipped", func(t *testing.T) {
// Only FieldTypeArray + ElementType=FieldTypeStruct triggers StructSchema validation.
s := NewSchema().WithName("c").
WithField(NewField().WithName("id").WithDataType(FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(NewField().WithName("tags").WithDataType(FieldTypeArray).WithElementType(FieldTypeInt32))
assert.NoError(t, s.Validate())
})
}
func TestSchemaValidateStructArray(t *testing.T) {
// valid schema passes
s := NewSchema().WithName("c").
WithField(NewField().WithName("id").WithDataType(FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(NewField().WithName("clips").
WithDataType(FieldTypeArray).WithElementType(FieldTypeStruct).
WithMaxCapacity(32).
WithStructSchema(NewStructSchema().
WithField(NewField().WithName("tag").WithDataType(FieldTypeVarChar).WithMaxLength(64)).
WithField(NewField().WithName("emb").WithDataType(FieldTypeFloatVector).WithDim(8))))
assert.NoError(t, s.Validate())
// schema with invalid struct sub-field fails
bad := NewSchema().WithName("c").
WithField(NewField().WithName("id").WithDataType(FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(NewField().WithName("clips").
WithDataType(FieldTypeArray).WithElementType(FieldTypeStruct).
WithStructSchema(NewStructSchema().
WithField(NewField().WithName("a").WithDataType(FieldTypeStruct))))
assert.Error(t, bad.Validate())
}
func TestFieldWithStructSchema(t *testing.T) {
// Create a struct schema
structSchema := NewStructSchema().
client/entity/schema.go
+71 -8
View File
@@ -17,6 +17,7 @@
package entity
import (
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@@ -154,17 +155,28 @@ func (s *Schema) ProtoMessage() *schemapb.CollectionSchema {
Description: field.Description,
TypeParams: MapKvPairs(field.TypeParams),
}
// max_capacity declared on the parent struct field must be carried onto each sub-field's
// type params — the server validates it per sub-field on insert.
parentMaxCap, hasParentMaxCap := field.TypeParams[TypeParamMaxCapacity]
if field.StructSchema != nil {
f.Fields = lo.Map(field.StructSchema.Fields, func(field *Field, _ int) *schemapb.FieldSchema {
f.Fields = lo.Map(field.StructSchema.Fields, func(sub *Field, _ int) *schemapb.FieldSchema {
// translate to ArrayStruct
f := field.ProtoMessage()
f.ElementType = f.DataType
if typeutil.IsVectorType(f.DataType) {
f.DataType = schemapb.DataType_ArrayOfVector
p := sub.ProtoMessage()
p.ElementType = p.DataType
if typeutil.IsVectorType(p.DataType) {
p.DataType = schemapb.DataType_ArrayOfVector
} else {
f.DataType = schemapb.DataType_Array
p.DataType = schemapb.DataType_Array
}
return f
if hasParentMaxCap {
if _, ok := sub.TypeParams[TypeParamMaxCapacity]; !ok {
p.TypeParams = append(p.TypeParams, &commonpb.KeyValuePair{
Key: TypeParamMaxCapacity,
Value: parentMaxCap,
})
}
}
return p
})
}
return f, true
@@ -181,7 +193,7 @@ func (s *Schema) ReadProto(p *schemapb.CollectionSchema) *Schema {
s.ExternalSource = p.GetExternalSource()
s.ExternalSpec = p.GetExternalSpec()
// fields
s.Fields = make([]*Field, 0, len(p.GetFields()))
s.Fields = make([]*Field, 0, len(p.GetFields())+len(p.GetStructArrayFields()))
for _, fp := range p.GetFields() {
field := NewField().ReadProto(fp)
if fp.GetAutoID() {
@@ -192,6 +204,10 @@ func (s *Schema) ReadProto(p *schemapb.CollectionSchema) *Schema {
}
s.Fields = append(s.Fields, field)
}
// struct array fields
for _, sp := range p.GetStructArrayFields() {
s.Fields = append(s.Fields, structArrayFieldFromProto(sp))
}
// functions
s.Functions = lo.Map(p.GetFunctions(), func(fn *schemapb.FunctionSchema, _ int) *Function {
return NewFunction().ReadProto(fn)
@@ -200,6 +216,53 @@ func (s *Schema) ReadProto(p *schemapb.CollectionSchema) *Schema {
return s
}
// structArrayFieldFromProto reverses the ProtoMessage transformation for struct array fields.
// Server returns struct sub-fields with DataType_Array / DataType_ArrayOfVector and the original
// element type carried in ElementType. We restore the user-facing Field where DataType is the
// original (scalar or vector) type.
func structArrayFieldFromProto(p *schemapb.StructArrayFieldSchema) *Field {
structSchema := NewStructSchema()
for _, sf := range p.GetFields() {
field := NewField().ReadProto(sf)
// unwrap Array/ArrayOfVector wrapper added by ProtoMessage()
switch sf.GetDataType() {
case schemapb.DataType_Array, schemapb.DataType_ArrayOfVector:
field.DataType = FieldType(sf.GetElementType())
field.ElementType = 0
}
structSchema.WithField(field)
}
return &Field{
ID: p.GetFieldID(),
Name: p.GetName(),
Description: p.GetDescription(),
DataType: FieldTypeArray,
ElementType: FieldTypeStruct,
TypeParams: KvPairsMap(p.GetTypeParams()),
StructSchema: structSchema,
}
}
// Validate performs client-side sanity checks on the schema. Currently enforces struct-array
// sub-field rules; may grow over time. Callers should invoke this before CreateCollection; the
// CreateCollection client path invokes it automatically so users opt in by default.
func (s *Schema) Validate() error {
if s == nil {
return errors.New("nil schema")
}
for _, field := range s.Fields {
if field == nil {
return errors.New("schema contains nil field")
}
if field.DataType == FieldTypeArray && field.ElementType == FieldTypeStruct {
if err := field.StructSchema.Validate(field.Name); err != nil {
return err
}
}
}
return nil
}
// PKFieldName returns pk field name for this schemapb.
func (s *Schema) PKFieldName() string {
if s.pkField == nil {
client/entity/schema_test.go
+45
View File
@@ -200,6 +200,51 @@ func (s *SchemaSuite) TestMultipleStructArrayFields() {
s.Equal(2, len(p.GetStructArrayFields()[1].GetFields()))
}
func (s *SchemaSuite) TestStructArrayFieldRoundTrip() {
structSchema := NewStructSchema().
WithField(NewField().WithName("clip_str").WithDataType(FieldTypeVarChar).WithMaxLength(256)).
WithField(NewField().WithName("clip_emb").WithDataType(FieldTypeFloatVector).WithDim(8))
schema := NewSchema().
WithName("rt").
WithField(NewField().WithName("id").WithDataType(FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(NewField().WithName("vec").WithDataType(FieldTypeFloatVector).WithDim(8)).
WithField(NewField().
WithName("clips").
WithDataType(FieldTypeArray).
WithElementType(FieldTypeStruct).
WithMaxCapacity(16).
WithStructSchema(structSchema))
p := schema.ProtoMessage()
s.Equal(2, len(p.GetFields()))
s.Equal(1, len(p.GetStructArrayFields()))
got := (&Schema{}).ReadProto(p)
// 3 logical fields including the struct array
s.Equal(3, len(got.Fields))
var clips *Field
for _, f := range got.Fields {
if f.Name == "clips" {
clips = f
break
}
}
s.Require().NotNil(clips)
s.Equal(FieldTypeArray, clips.DataType)
s.Equal(FieldTypeStruct, clips.ElementType)
s.Require().NotNil(clips.StructSchema)
s.Equal(2, len(clips.StructSchema.Fields))
// Sub-fields should be restored to their original types, not Array/ArrayOfVector
s.Equal(FieldTypeVarChar, clips.StructSchema.Fields[0].DataType)
s.Equal(FieldTypeFloatVector, clips.StructSchema.Fields[1].DataType)
dim, err := clips.StructSchema.Fields[1].GetDim()
s.NoError(err)
s.EqualValues(8, dim)
}
func TestSchema(t *testing.T) {
suite.Run(t, new(SchemaSuite))
}
client/entity/vectors.go
+111
View File
@@ -149,3 +149,114 @@ func (iv Int8Vector) Serialize() []byte {
func (iv Int8Vector) FieldType() FieldType {
return FieldTypeInt8Vector
}
// serializeVectorArray concatenates inner vector bytes for EmbList placeholder values.
func serializeVectorArray[V Vector](vs []V) []byte {
if len(vs) == 0 {
return nil
}
out := make([]byte, 0, len(vs)*len(vs[0].Serialize()))
for _, v := range vs {
out = append(out, v.Serialize()...)
}
return out
}
// FloatVectorArray groups multiple float vectors into one search query, used for MAX_SIM-style
// search against ArrayOfVector sub-fields of struct array (e.g. anns_field="clips[clip_emb]").
// Each call to Search receives one or more `Vector` items; pass FloatVectorArray instead of
// FloatVector when you want each query slot to carry a list of vectors (the EmbeddingList
// equivalent in pymilvus).
type FloatVectorArray []FloatVector
// Dim returns the dim of the inner vectors (assumed uniform).
func (fa FloatVectorArray) Dim() int {
if len(fa) == 0 {
return 0
}
return fa[0].Dim()
}
// FieldType returns the underlying float vector field type so the read path can pick the right
// EmbList placeholder type.
func (fa FloatVectorArray) FieldType() FieldType {
return FieldTypeFloatVector
}
// Serialize concatenates the bytes of each inner vector, matching the format the server expects
// for EmbListFloatVector placeholder values.
func (fa FloatVectorArray) Serialize() []byte {
return serializeVectorArray(fa)
}
// Float16VectorArray groups multiple float16 vectors for EmbListFloat16Vector search.
type Float16VectorArray []Float16Vector
func (fa Float16VectorArray) Dim() int {
if len(fa) == 0 {
return 0
}
return fa[0].Dim()
}
func (fa Float16VectorArray) FieldType() FieldType {
return FieldTypeFloat16Vector
}
func (fa Float16VectorArray) Serialize() []byte {
return serializeVectorArray(fa)
}
// BFloat16VectorArray groups multiple bfloat16 vectors for EmbListBFloat16Vector search.
type BFloat16VectorArray []BFloat16Vector
func (fa BFloat16VectorArray) Dim() int {
if len(fa) == 0 {
return 0
}
return fa[0].Dim()
}
func (fa BFloat16VectorArray) FieldType() FieldType {
return FieldTypeBFloat16Vector
}
func (fa BFloat16VectorArray) Serialize() []byte {
return serializeVectorArray(fa)
}
// BinaryVectorArray groups multiple binary vectors for EmbListBinaryVector search.
type BinaryVectorArray []BinaryVector
func (fa BinaryVectorArray) Dim() int {
if len(fa) == 0 {
return 0
}
return fa[0].Dim()
}
func (fa BinaryVectorArray) FieldType() FieldType {
return FieldTypeBinaryVector
}
func (fa BinaryVectorArray) Serialize() []byte {
return serializeVectorArray(fa)
}
// Int8VectorArray groups multiple int8 vectors for EmbListInt8Vector search.
type Int8VectorArray []Int8Vector
func (fa Int8VectorArray) Dim() int {
if len(fa) == 0 {
return 0
}
return fa[0].Dim()
}
func (fa Int8VectorArray) FieldType() FieldType {
return FieldTypeInt8Vector
}
func (fa Int8VectorArray) Serialize() []byte {
return serializeVectorArray(fa)
}
client/entity/vectors_test.go
+84
View File
@@ -104,3 +104,87 @@ func TestVectors(t *testing.T) {
assert.Equal(t, dim, len(iv.Serialize()))
})
}
func TestVectorArrays(t *testing.T) {
const dim = 4
const rows = 3
t.Run("float vector array", func(t *testing.T) {
fa := FloatVectorArray{
FloatVector{1, 2, 3, 4},
FloatVector{5, 6, 7, 8},
FloatVector{9, 10, 11, 12},
}
assert.Equal(t, FieldTypeFloatVector, fa.FieldType())
assert.Equal(t, dim, fa.Dim())
// 3 vectors * dim floats * 4 bytes/float
assert.Equal(t, rows*dim*4, len(fa.Serialize()))
var empty FloatVectorArray
assert.Equal(t, 0, empty.Dim())
assert.Nil(t, empty.Serialize())
})
t.Run("float16 vector array", func(t *testing.T) {
raw := make([]byte, dim*2)
fa := Float16VectorArray{Float16Vector(raw), Float16Vector(raw)}
assert.Equal(t, FieldTypeFloat16Vector, fa.FieldType())
assert.Equal(t, dim, fa.Dim())
assert.Equal(t, 2*dim*2, len(fa.Serialize()))
})
t.Run("bfloat16 vector array", func(t *testing.T) {
raw := make([]byte, dim*2)
fa := BFloat16VectorArray{BFloat16Vector(raw), BFloat16Vector(raw)}
assert.Equal(t, FieldTypeBFloat16Vector, fa.FieldType())
assert.Equal(t, dim, fa.Dim())
assert.Equal(t, 2*dim*2, len(fa.Serialize()))
})
t.Run("binary vector array", func(t *testing.T) {
// binary dim is bits; use 2 bytes -> 16 bits.
raw := make([]byte, 2)
fa := BinaryVectorArray{BinaryVector(raw), BinaryVector(raw)}
assert.Equal(t, FieldTypeBinaryVector, fa.FieldType())
assert.Equal(t, 16, fa.Dim())
assert.Equal(t, 2*2, len(fa.Serialize()))
})
t.Run("int8 vector array", func(t *testing.T) {
raw := make([]int8, dim)
fa := Int8VectorArray{Int8Vector(raw), Int8Vector(raw), Int8Vector(raw)}
assert.Equal(t, FieldTypeInt8Vector, fa.FieldType())
assert.Equal(t, dim, fa.Dim())
assert.Equal(t, rows*dim, len(fa.Serialize()))
})
t.Run("empty arrays report zero dim and nil serialize", func(t *testing.T) {
assert.Equal(t, 0, FloatVectorArray(nil).Dim())
assert.Equal(t, 0, Float16VectorArray(nil).Dim())
assert.Equal(t, 0, BFloat16VectorArray(nil).Dim())
assert.Equal(t, 0, BinaryVectorArray(nil).Dim())
assert.Equal(t, 0, Int8VectorArray(nil).Dim())
assert.Nil(t, FloatVectorArray(nil).Serialize())
assert.Nil(t, Float16VectorArray(nil).Serialize())
assert.Nil(t, BFloat16VectorArray(nil).Serialize())
assert.Nil(t, BinaryVectorArray(nil).Serialize())
assert.Nil(t, Int8VectorArray(nil).Serialize())
})
}
func TestVectorFieldType(t *testing.T) {
// FieldType() for each Vector type is used by vector2Placeholder to pick the placeholder slot.
assert.Equal(t, FieldTypeFloatVector, FloatVector{}.FieldType())
assert.Equal(t, FieldTypeFloat16Vector, Float16Vector{}.FieldType())
assert.Equal(t, FieldTypeBFloat16Vector, BFloat16Vector{}.FieldType())
assert.Equal(t, FieldTypeBinaryVector, BinaryVector{}.FieldType())
assert.Equal(t, FieldTypeInt8Vector, Int8Vector{}.FieldType())
}
func TestTextVector(t *testing.T) {
txt := Text("hello")
assert.Equal(t, 0, txt.Dim())
assert.Equal(t, FieldTypeVarChar, txt.FieldType())
assert.Equal(t, []byte("hello"), txt.Serialize())
}
client/milvusclient/collection.go
+6
View File
@@ -29,6 +29,12 @@ import (
// CreateCollection is the API for create a collection in Milvus.
func (c *Client) CreateCollection(ctx context.Context, option CreateCollectionOption, callOptions ...grpc.CallOption) error {
// Client-side schema validation: options that expose Validate() get a pre-flight sanity check.
if v, ok := option.(interface{ Validate() error }); ok {
if err := v.Validate(); err != nil {
return err
}
}
req := option.Request()
err := c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
client/milvusclient/collection_options.go
+9
View File
@@ -122,6 +122,15 @@ func (opt *createCollectionOption) WithNumPartitions(numPartitions int64) *creat
return opt
}
// Validate runs client-side sanity checks against the user-provided schema. Invoked automatically
// from Client.CreateCollection via interface assertion.
func (opt *createCollectionOption) Validate() error {
if opt.schema == nil {
return nil
}
return opt.schema.Validate()
}
func (opt *createCollectionOption) Request() *milvuspb.CreateCollectionRequest {
// fast create collection
if opt.isFast {
client/milvusclient/read_option_test.go
+25
View File
@@ -142,6 +142,31 @@ func (s *SearchOptionSuite) TestPlaceHolder() {
input: []entity.Vector{entity.Text("abc")},
expectType: commonpb.PlaceholderType_VarChar,
},
{
tag: "emb_list_float",
input: []entity.Vector{entity.FloatVectorArray{entity.FloatVector([]float32{0.1, 0.2})}},
expectType: commonpb.PlaceholderType_EmbListFloatVector,
},
{
tag: "emb_list_fp16",
input: []entity.Vector{entity.Float16VectorArray{entity.Float16Vector([]byte{})}},
expectType: commonpb.PlaceholderType_EmbListFloat16Vector,
},
{
tag: "emb_list_bf16",
input: []entity.Vector{entity.BFloat16VectorArray{entity.BFloat16Vector([]byte{})}},
expectType: commonpb.PlaceholderType_EmbListBFloat16Vector,
},
{
tag: "emb_list_binary",
input: []entity.Vector{entity.BinaryVectorArray{entity.BinaryVector([]byte{})}},
expectType: commonpb.PlaceholderType_EmbListBinaryVector,
},
{
tag: "emb_list_int8",
input: []entity.Vector{entity.Int8VectorArray{entity.Int8Vector([]int8{})}},
expectType: commonpb.PlaceholderType_EmbListInt8Vector,
},
{
tag: "non_supported",
input: []entity.Vector{nonSupportData{}},
client/milvusclient/read_options.go
+10
View File
@@ -470,6 +470,16 @@ func vector2Placeholder(vectors []entity.Vector) (*commonpb.PlaceholderValue, er
placeHolderType = commonpb.PlaceholderType_Int8Vector
case entity.Text:
placeHolderType = commonpb.PlaceholderType_VarChar
case entity.FloatVectorArray:
placeHolderType = commonpb.PlaceholderType_EmbListFloatVector
case entity.Float16VectorArray:
placeHolderType = commonpb.PlaceholderType_EmbListFloat16Vector
case entity.BFloat16VectorArray:
placeHolderType = commonpb.PlaceholderType_EmbListBFloat16Vector
case entity.BinaryVectorArray:
placeHolderType = commonpb.PlaceholderType_EmbListBinaryVector
case entity.Int8VectorArray:
placeHolderType = commonpb.PlaceholderType_EmbListInt8Vector
default:
return nil, errors.Newf("unsupported search data type: %T", vectors[0])
}
client/milvusclient/write_option_test.go
+149
View File
@@ -22,6 +22,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/client/v2/column"
"github.com/milvus-io/milvus/client/v2/entity"
)
@@ -46,6 +47,154 @@ func (s *ColumnBasedDataOptionSuite) NullableCompatible() {
s.ElementsMatch([]bool{true, true, true}, fd.GetValidData())
}
func (s *ColumnBasedDataOptionSuite) TestWithStructArrayColumn() {
dim := 4
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("clip_str").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64)).
WithField(entity.NewField().WithName("clip_emb").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim)))
collSchema := entity.NewSchema().WithName("c").
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("vec").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().
WithName("clips").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(16).
WithStructSchema(structSchema))
rows := []map[string]any{
{"clip_str": []string{"a", "b"}, "clip_emb": [][]float32{{0.1, 0.2, 0.3, 0.4}, {0.5, 0.6, 0.7, 0.8}}},
{"clip_str": []string{"c"}, "clip_emb": [][]float32{{1.0, 1.0, 1.0, 1.0}}},
}
opt := NewColumnBasedInsertOption("c").
WithInt64Column("id", []int64{1, 2}).
WithFloatVectorColumn("vec", dim, [][]float32{{0, 0, 0, 0}, {1, 1, 1, 1}}).
WithStructArrayColumn("clips", structSchema, rows)
coll := &entity.Collection{Schema: collSchema}
req, err := opt.InsertRequest(coll)
s.Require().NoError(err)
s.EqualValues(2, req.GetNumRows())
var clipsFD *schemapb.FieldData
for _, fd := range req.GetFieldsData() {
if fd.GetFieldName() == "clips" {
clipsFD = fd
break
}
}
s.Require().NotNil(clipsFD)
s.Equal(schemapb.DataType_ArrayOfStruct, clipsFD.GetType())
subs := clipsFD.GetStructArrays().GetFields()
s.Require().Equal(2, len(subs))
// Find each sub by name (order is not guaranteed by builder).
var strSub, embSub *schemapb.FieldData
for _, sub := range subs {
switch sub.GetFieldName() {
case "clip_str":
strSub = sub
case "clip_emb":
embSub = sub
}
}
s.Require().NotNil(strSub)
s.Require().NotNil(embSub)
s.Equal(schemapb.DataType_Array, strSub.GetType())
arr := strSub.GetScalars().GetArrayData().GetData()
s.Require().Equal(2, len(arr))
s.Equal([]string{"a", "b"}, arr[0].GetStringData().GetData())
s.Equal([]string{"c"}, arr[1].GetStringData().GetData())
s.Equal(schemapb.DataType_ArrayOfVector, embSub.GetType())
va := embSub.GetVectors().GetVectorArray()
s.Require().NotNil(va)
s.EqualValues(dim, va.GetDim())
s.Equal(schemapb.DataType_FloatVector, va.GetElementType())
s.Require().Equal(2, len(va.GetData()))
s.EqualValues(2*dim, len(va.GetData()[0].GetFloatVector().GetData()))
s.EqualValues(1*dim, len(va.GetData()[1].GetFloatVector().GetData()))
}
func (s *ColumnBasedDataOptionSuite) TestWithStructArrayColumnDeferredError() {
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("clip_str").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64))
// Pass rows with missing sub-field — builder must NOT panic; error surfaces on InsertRequest.
s.NotPanics(func() {
opt := NewColumnBasedInsertOption("c").
WithInt64Column("id", []int64{1}).
WithStructArrayColumn("clips", structSchema, []map[string]any{{"wrong_key": []string{"a"}}})
coll := &entity.Collection{Schema: entity.NewSchema().WithName("c").
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true))}
_, err := opt.InsertRequest(coll)
s.Require().Error(err)
// UpsertRequest must also surface the deferred error instead of panicking.
_, upsertErr := opt.UpsertRequest(coll)
s.Require().Error(upsertErr)
})
}
func (s *ColumnBasedDataOptionSuite) TestWithStructArrayColumnNilSchema() {
// nil schema must be rejected at build time (deferred).
opt := NewColumnBasedInsertOption("c").
WithStructArrayColumn("clips", nil, nil)
coll := &entity.Collection{Schema: entity.NewSchema().WithName("c")}
_, err := opt.InsertRequest(coll)
s.Error(err)
}
func (s *ColumnBasedDataOptionSuite) TestNewStructSubColumnAllSupportedTypes() {
// All scalar and vector sub-field types supported by newStructSubColumn; each must produce
// a non-nil sub-column without error. Vector types also require a valid dim.
dim := 8
cases := []*entity.Field{
entity.NewField().WithName("b").WithDataType(entity.FieldTypeBool),
entity.NewField().WithName("i8").WithDataType(entity.FieldTypeInt8),
entity.NewField().WithName("i16").WithDataType(entity.FieldTypeInt16),
entity.NewField().WithName("i32").WithDataType(entity.FieldTypeInt32),
entity.NewField().WithName("i64").WithDataType(entity.FieldTypeInt64),
entity.NewField().WithName("f").WithDataType(entity.FieldTypeFloat),
entity.NewField().WithName("d").WithDataType(entity.FieldTypeDouble),
entity.NewField().WithName("s").WithDataType(entity.FieldTypeVarChar).WithMaxLength(16),
entity.NewField().WithName("str").WithDataType(entity.FieldTypeString),
entity.NewField().WithName("fv").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim)),
entity.NewField().WithName("fp16").WithDataType(entity.FieldTypeFloat16Vector).WithDim(int64(dim)),
entity.NewField().WithName("bf16").WithDataType(entity.FieldTypeBFloat16Vector).WithDim(int64(dim)),
entity.NewField().WithName("bv").WithDataType(entity.FieldTypeBinaryVector).WithDim(int64(dim)),
entity.NewField().WithName("i8v").WithDataType(entity.FieldTypeInt8Vector).WithDim(int64(dim)),
}
for _, f := range cases {
c, err := newStructSubColumn(f)
s.Require().NoError(err, "type %v", f.DataType)
s.NotNil(c)
}
}
func (s *ColumnBasedDataOptionSuite) TestNewStructSubColumnErrors() {
// Unsupported data type in a struct sub-field must error.
_, err := newStructSubColumn(entity.NewField().WithName("bad").WithDataType(entity.FieldTypeJSON))
s.Error(err)
// Vector sub-fields without dim must surface GetDim's error.
for _, dt := range []entity.FieldType{
entity.FieldTypeFloatVector,
entity.FieldTypeFloat16Vector,
entity.FieldTypeBFloat16Vector,
entity.FieldTypeBinaryVector,
entity.FieldTypeInt8Vector,
} {
_, err := newStructSubColumn(entity.NewField().WithName("no_dim").WithDataType(dt))
s.Error(err, "type %v", dt)
}
}
func TestRowBasedDataOption(t *testing.T) {
suite.Run(t, new(ColumnBasedDataOptionSuite))
}
client/milvusclient/write_options.go
+113
View File
@@ -53,6 +53,10 @@ type columnBasedDataOption struct {
partitionName string
columns []column.Column
partialUpdate bool
// deferredErr captures construction-time errors from builder helpers (e.g. WithStructArrayColumn)
// so they surface on InsertRequest/UpsertRequest rather than panicking in the chain.
deferredErr error
}
func (opt *columnBasedDataOption) WriteBackPKs(_ *entity.Schema, _ column.Column) error {
@@ -262,6 +266,109 @@ func (opt *columnBasedDataOption) WithInt8VectorColumn(colName string, dim int,
return opt.WithColumns(column)
}
// WithStructArrayColumn appends a struct-array column built from a row-based representation,
// inferring the per-sub-field array type from the corresponding field in `structSchema`.
//
// `rows` is a per-collection-row list; each entry is a map keyed by sub-field name. The value
// for a scalar sub-field must be `[]<T>` (e.g. []int32, []string); the value for a vector
// sub-field must be `[][]float32` / `[][]byte` / `[][]int8` matching the vector type.
//
// Example:
//
// structSchema := entity.NewStructSchema().
// WithField(entity.NewField().WithName("clip_str").WithDataType(entity.FieldTypeVarChar).WithMaxLength(256)).
// WithField(entity.NewField().WithName("clip_emb").WithDataType(entity.FieldTypeFloatVector).WithDim(8))
// rows := []map[string]any{
// {"clip_str": []string{"a", "b"}, "clip_emb": [][]float32{v1, v2}},
// {"clip_str": []string{"c"}, "clip_emb": [][]float32{v3}},
// }
// opt.WithStructArrayColumn("clips", structSchema, rows)
func (opt *columnBasedDataOption) WithStructArrayColumn(colName string, structSchema *entity.StructSchema, rows []map[string]any) *columnBasedDataOption {
col, err := buildStructArrayColumn(colName, structSchema, rows)
if err != nil {
// Defer error reporting to InsertRequest/UpsertRequest so the builder chain stays valid.
if opt.deferredErr == nil {
opt.deferredErr = errors.Wrapf(err, "WithStructArrayColumn(%q)", colName)
}
return opt
}
return opt.WithColumns(col)
}
func buildStructArrayColumn(colName string, structSchema *entity.StructSchema, rows []map[string]any) (column.Column, error) {
if structSchema == nil {
return nil, errors.New("structSchema is required for WithStructArrayColumn")
}
subColumns := make([]column.Column, 0, len(structSchema.Fields))
for _, sub := range structSchema.Fields {
subColumn, err := newStructSubColumn(sub)
if err != nil {
return nil, err
}
subColumns = append(subColumns, subColumn)
}
structCol := column.NewColumnStructArray(colName, subColumns)
for i, row := range rows {
if err := structCol.AppendValue(row); err != nil {
return nil, errors.Wrapf(err, "row %d", i)
}
}
return structCol, nil
}
func newStructSubColumn(field *entity.Field) (column.Column, error) {
switch field.DataType {
case entity.FieldTypeBool:
return column.NewColumnBoolArray(field.Name, nil), nil
case entity.FieldTypeInt8:
return column.NewColumnInt8Array(field.Name, nil), nil
case entity.FieldTypeInt16:
return column.NewColumnInt16Array(field.Name, nil), nil
case entity.FieldTypeInt32:
return column.NewColumnInt32Array(field.Name, nil), nil
case entity.FieldTypeInt64:
return column.NewColumnInt64Array(field.Name, nil), nil
case entity.FieldTypeFloat:
return column.NewColumnFloatArray(field.Name, nil), nil
case entity.FieldTypeDouble:
return column.NewColumnDoubleArray(field.Name, nil), nil
case entity.FieldTypeVarChar, entity.FieldTypeString:
return column.NewColumnVarCharArray(field.Name, nil), nil
case entity.FieldTypeFloatVector:
dim, err := field.GetDim()
if err != nil {
return nil, errors.Wrapf(err, "sub-field %q", field.Name)
}
return column.NewColumnFloatVectorArray(field.Name, int(dim), nil), nil
case entity.FieldTypeFloat16Vector:
dim, err := field.GetDim()
if err != nil {
return nil, errors.Wrapf(err, "sub-field %q", field.Name)
}
return column.NewColumnFloat16VectorArray(field.Name, int(dim), nil), nil
case entity.FieldTypeBFloat16Vector:
dim, err := field.GetDim()
if err != nil {
return nil, errors.Wrapf(err, "sub-field %q", field.Name)
}
return column.NewColumnBFloat16VectorArray(field.Name, int(dim), nil), nil
case entity.FieldTypeBinaryVector:
dim, err := field.GetDim()
if err != nil {
return nil, errors.Wrapf(err, "sub-field %q", field.Name)
}
return column.NewColumnBinaryVectorArray(field.Name, int(dim), nil), nil
case entity.FieldTypeInt8Vector:
dim, err := field.GetDim()
if err != nil {
return nil, errors.Wrapf(err, "sub-field %q", field.Name)
}
return column.NewColumnInt8VectorArray(field.Name, int(dim), nil), nil
default:
return nil, errors.Newf("unsupported struct sub-field type %v for field %q", field.DataType, field.Name)
}
}
func (opt *columnBasedDataOption) WithPartition(partitionName string) *columnBasedDataOption {
opt.partitionName = partitionName
return opt
@@ -277,6 +384,9 @@ func (opt *columnBasedDataOption) CollectionName() string {
}
func (opt *columnBasedDataOption) InsertRequest(coll *entity.Collection) (*milvuspb.InsertRequest, error) {
if opt.deferredErr != nil {
return nil, opt.deferredErr
}
fieldsData, rowNum, err := opt.processInsertColumns(coll.Schema, opt.columns...)
if err != nil {
return nil, err
@@ -291,6 +401,9 @@ func (opt *columnBasedDataOption) InsertRequest(coll *entity.Collection) (*milvu
}
func (opt *columnBasedDataOption) UpsertRequest(coll *entity.Collection) (*milvuspb.UpsertRequest, error) {
if opt.deferredErr != nil {
return nil, opt.deferredErr
}
fieldsData, rowNum, err := opt.processInsertColumns(coll.Schema, opt.columns...)
if err != nil {
return nil, err
tests/go_client/testcases/helper/struct_array_element_helper.go
+555
View File
@@ -0,0 +1,555 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helper
import (
"fmt"
"math/rand"
"github.com/milvus-io/milvus/client/v2/entity"
)
// Mirrors constants from tests/python_client/milvus_client/
// test_milvus_client_struct_array_element_query.py and ..._element_search.py.
const (
StructAElemPrefix = "struct_elem"
StructAElemDim = 128
StructAElemCapacity = 10
StructAElemSealedNb = 200 // python uses default_nb=3000; smaller for Go SDK runs
StructAElemGrowingNb = 50
StructAElemMaxStrLen = 65535
StructAElemMaxColorLen = 128
)
// COLORS and CATEGORIES match the Python fixtures so element_filter expressions and ground-truth
// comparisons stay identical.
var (
StructAElemColors = []string{"Red", "Blue", "Green"}
StructAElemCategories = []string{"A", "B", "C", "D"}
StructAElemSizes = []string{"S", "M", "L", "XL"}
)
// StructAElementSchemaOption controls which sub-fields are present in the canonical structA schema.
// Defaults match the union of sub-fields used by the Python tests so a single helper covers all.
type StructAElementSchemaOption struct {
Dim int
Capacity int
IncludeDocInt bool
IncludeDocVChar bool // doc_varchar at the row level
IncludeStrVal bool
IncludeFloatVal bool
IncludeCategory bool
IncludeSize bool // adds a "size" VarChar sub-field used by element_search tests
CollectionName string
StructFieldName string // default "structA"
NormalVectorName string // default "normal_vector"
}
// DefaultStructAElementSchemaOption returns the union schema (every sub-field present), suitable for
// 90 % of element-query/search tests.
func DefaultStructAElementSchemaOption(name string) StructAElementSchemaOption {
return StructAElementSchemaOption{
Dim: StructAElemDim,
Capacity: StructAElemCapacity,
IncludeDocInt: true,
IncludeDocVChar: true,
IncludeStrVal: true,
IncludeFloatVal: true,
IncludeCategory: true,
CollectionName: name,
StructFieldName: "structA",
NormalVectorName: "normal_vector",
}
}
// CreateStructAElementSchema builds the canonical schema. Returns the entity.Schema and the inner
// StructSchema (the latter is needed by WithStructArrayColumn).
func CreateStructAElementSchema(opt StructAElementSchemaOption) (*entity.Schema, *entity.StructSchema) {
if opt.Dim == 0 {
opt.Dim = StructAElemDim
}
if opt.Capacity == 0 {
opt.Capacity = StructAElemCapacity
}
if opt.StructFieldName == "" {
opt.StructFieldName = "structA"
}
if opt.NormalVectorName == "" {
opt.NormalVectorName = "normal_vector"
}
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("embedding").
WithDataType(entity.FieldTypeFloatVector).WithDim(int64(opt.Dim))).
WithField(entity.NewField().WithName("int_val").
WithDataType(entity.FieldTypeInt64))
if opt.IncludeStrVal {
structSchema.WithField(entity.NewField().WithName("str_val").
WithDataType(entity.FieldTypeVarChar).WithMaxLength(StructAElemMaxStrLen))
}
if opt.IncludeFloatVal {
structSchema.WithField(entity.NewField().WithName("float_val").
WithDataType(entity.FieldTypeFloat))
}
structSchema.WithField(entity.NewField().WithName("color").
WithDataType(entity.FieldTypeVarChar).WithMaxLength(StructAElemMaxColorLen))
if opt.IncludeCategory {
structSchema.WithField(entity.NewField().WithName("category").
WithDataType(entity.FieldTypeVarChar).WithMaxLength(StructAElemMaxColorLen))
}
if opt.IncludeSize {
structSchema.WithField(entity.NewField().WithName("size").
WithDataType(entity.FieldTypeVarChar).WithMaxLength(StructAElemMaxColorLen))
}
schema := entity.NewSchema().WithName(opt.CollectionName).
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true))
if opt.IncludeDocInt {
schema.WithField(entity.NewField().WithName("doc_int").WithDataType(entity.FieldTypeInt64))
}
if opt.IncludeDocVChar {
schema.WithField(entity.NewField().WithName("doc_varchar").
WithDataType(entity.FieldTypeVarChar).WithMaxLength(256))
}
schema.WithField(entity.NewField().WithName(opt.NormalVectorName).
WithDataType(entity.FieldTypeFloatVector).WithDim(int64(opt.Dim)))
schema.WithField(entity.NewField().WithName(opt.StructFieldName).
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(int64(opt.Capacity)).
WithStructSchema(structSchema))
return schema, structSchema
}
// StructAElement represents one struct element in a row. Used both as ground-truth source and
// as input to per-row insert generators.
type StructAElement struct {
Embedding []float32
IntVal int64
StrVal string
FloatVal float32
Color string
Category string
Size string
}
// StructARow represents one row including doc-level fields. Returned by generators and used by
// ground-truth filters.
type StructARow struct {
ID int64
DocInt int64
DocVarChar string
NormalVector []float32
StructA []StructAElement
}
// StructAElementDataset bundles columns ready for insert plus the structured rows for ground truth.
type StructAElementDataset struct {
Rows []StructARow
Opt StructAElementSchemaOption
}
// SeedVector mirrors python `_seed_vector(seed)` — deterministic uniform random float vector.
// Python's helper also normalises so we keep that for embedding/cosine math parity.
func SeedVector(seed int64, dim int) []float32 {
r := rand.New(rand.NewSource(seed))
v := make([]float32, dim)
var norm float64
for i := range v {
v[i] = r.Float32()
norm += float64(v[i]) * float64(v[i])
}
if norm <= 0 {
return v
}
inv := 1.0 / float32sqrt(norm)
for i := range v {
v[i] *= inv
}
return v
}
func float32sqrt(x float64) float32 {
// avoid pulling math just for one sqrt at this size
z := x
for i := 0; i < 16; i++ {
z = 0.5 * (z + x/z)
}
return float32(z)
}
// GenerateStructAElementData mirrors the deterministic generator used by the python tests:
// - num_elems = random.Random(i).randint(3, 8) (python inclusive on both ends)
// - int_val = i*100 + j
// - str_val = f"row_{i}_elem_{j}"
// - float_val = i + j*0.1
// - color = COLORS[j % 3]
// - category = CATEGORIES[(i+j) % 4]
// - embedding = SeedVector(i*1000 + j)
func GenerateStructAElementData(nb int, startID int64, opt StructAElementSchemaOption) StructAElementDataset {
if opt.Dim == 0 {
opt.Dim = StructAElemDim
}
rows := make([]StructARow, 0, nb)
for i := int64(0); i < int64(nb); i++ {
id := startID + i
// emulate python random.Random(id).randint(3, 8) using a small PRNG seeded by id
r := rand.New(rand.NewSource(id))
numElems := 3 + r.Intn(6) // 3..8 inclusive
elems := make([]StructAElement, numElems)
for j := 0; j < numElems; j++ {
elems[j] = StructAElement{
Embedding: SeedVector(id*1000+int64(j), opt.Dim),
IntVal: id*100 + int64(j),
StrVal: fmt.Sprintf("row_%d_elem_%d", id, j),
FloatVal: float32(id) + float32(j)*0.1,
Color: StructAElemColors[j%3],
Category: StructAElemCategories[(int(id)+j)%4],
Size: StructAElemSizes[(int(id)+j)%4],
}
}
rows = append(rows, StructARow{
ID: id,
DocInt: id,
DocVarChar: fmt.Sprintf("cat_%d", id%10),
NormalVector: SeedVector(id+999999, opt.Dim),
StructA: elems,
})
}
return StructAElementDataset{Rows: rows, Opt: opt}
}
// ToInsertColumns returns the parallel column slices needed by WithStructArrayColumn etc.
//
// - ids, normalVectors are always returned
// - structRows is the row-keyed map[string]any payload to feed WithStructArrayColumn
// - docInts / docVChars are returned (zero values if not in schema) — caller uses based on opt
func (d StructAElementDataset) ToInsertColumns() (ids []int64, normalVectors [][]float32, docInts []int64, docVChars []string, structRows []map[string]any) {
ids = make([]int64, len(d.Rows))
normalVectors = make([][]float32, len(d.Rows))
docInts = make([]int64, len(d.Rows))
docVChars = make([]string, len(d.Rows))
structRows = make([]map[string]any, len(d.Rows))
for i, r := range d.Rows {
ids[i] = r.ID
normalVectors[i] = r.NormalVector
docInts[i] = r.DocInt
docVChars[i] = r.DocVarChar
structRows[i] = elementsToRow(r.StructA, d.Opt)
}
return
}
func elementsToRow(elements []StructAElement, opt StructAElementSchemaOption) map[string]any {
embs := make([][]float32, len(elements))
intVals := make([]int64, len(elements))
strVals := make([]string, len(elements))
floatVals := make([]float32, len(elements))
colors := make([]string, len(elements))
cats := make([]string, len(elements))
sizes := make([]string, len(elements))
for j, e := range elements {
embs[j] = e.Embedding
intVals[j] = e.IntVal
strVals[j] = e.StrVal
floatVals[j] = e.FloatVal
colors[j] = e.Color
cats[j] = e.Category
sizes[j] = e.Size
}
row := map[string]any{
"embedding": embs,
"int_val": intVals,
"color": colors,
}
if opt.IncludeStrVal {
row["str_val"] = strVals
}
if opt.IncludeFloatVal {
row["float_val"] = floatVals
}
if opt.IncludeCategory {
row["category"] = cats
}
if opt.IncludeSize {
row["size"] = sizes
}
return row
}
// MakeRow is a row builder used by Python `_make_row(row_id, struct_elements)` controlled-data
// tests. struct_elements only need to set fields the test cares about; missing fields default to
// safe values (color="Red", str_val=auto-generated, embedding=seeded).
func MakeRow(rowID int64, opt StructAElementSchemaOption, structElements []StructAElement) StructARow {
if opt.Dim == 0 {
opt.Dim = StructAElemDim
}
elems := make([]StructAElement, len(structElements))
for j, e := range structElements {
ej := e
if len(ej.Embedding) == 0 {
ej.Embedding = SeedVector(rowID*1000+int64(j), opt.Dim)
}
if ej.Color == "" {
ej.Color = "Red"
}
if ej.StrVal == "" {
ej.StrVal = fmt.Sprintf("r%d_e%d", rowID, j)
}
elems[j] = ej
}
return StructARow{
ID: rowID,
DocInt: rowID,
DocVarChar: fmt.Sprintf("cat_%d", rowID%10),
NormalVector: SeedVector(rowID+999999, opt.Dim),
StructA: elems,
}
}
// MakeInertRow creates a row that does NOT match common element_filter conditions. Used by the
// python correctness tests as background fill so element_filter results are unambiguous.
func MakeInertRow(rowID int64, opt StructAElementSchemaOption) StructARow {
if opt.Dim == 0 {
opt.Dim = StructAElemDim
}
return StructARow{
ID: rowID,
DocInt: 9000000 + rowID,
DocVarChar: "inert",
NormalVector: SeedVector(rowID+999999, opt.Dim),
StructA: []StructAElement{{
Embedding: SeedVector(rowID*1000, opt.Dim),
IntVal: 0,
StrVal: fmt.Sprintf("inert_%d", rowID),
Color: "Inert",
Category: "Inert",
FloatVal: 0,
}},
}
}
// RowsToColumns wraps ToInsertColumns for arbitrary StructARow slices that may have been built
// from MakeRow / MakeInertRow rather than the bulk generator.
func RowsToColumns(rows []StructARow, opt StructAElementSchemaOption) (ids []int64, normalVectors [][]float32, docInts []int64, docVChars []string, structRows []map[string]any) {
d := StructAElementDataset{Rows: rows, Opt: opt}
return d.ToInsertColumns()
}
// =============================================================================
// Ground truth helpers — port of gt_element_filter_query / gt_match_query / array_contains.
// =============================================================================
// GtElementFilter returns the set of row IDs for which at least one element in StructA satisfies
// elemFilterFn. If docFilterFn is non-nil it must also pass.
func GtElementFilter(data []StructARow, elemFilterFn func(StructAElement) bool, docFilterFn func(StructARow) bool) map[int64]struct{} {
ids := make(map[int64]struct{})
for _, row := range data {
if docFilterFn != nil && !docFilterFn(row) {
continue
}
for _, e := range row.StructA {
if elemFilterFn(e) {
ids[row.ID] = struct{}{}
break
}
}
}
return ids
}
// GtMatch covers MATCH_ALL / MATCH_ANY / MATCH_LEAST / MATCH_MOST / MATCH_EXACT. threshold is
// only used by the LEAST/MOST/EXACT variants.
func GtMatch(data []StructARow, matchType string, elemFilterFn func(StructAElement) bool, threshold int, docFilterFn func(StructARow) bool) map[int64]struct{} {
ids := make(map[int64]struct{})
for _, row := range data {
if docFilterFn != nil && !docFilterFn(row) {
continue
}
count := 0
for _, e := range row.StructA {
if elemFilterFn(e) {
count++
}
}
total := len(row.StructA)
var matched bool
switch matchType {
case "MATCH_ALL":
matched = count == total
case "MATCH_ANY":
matched = count >= 1
case "MATCH_LEAST":
matched = count >= threshold
case "MATCH_MOST":
matched = count <= threshold
case "MATCH_EXACT":
matched = count == threshold
}
if matched {
ids[row.ID] = struct{}{}
}
}
return ids
}
// GtArrayContains returns IDs whose StructA has at least one element where extractor(elem) ==
// target.
func GtArrayContains[T comparable](data []StructARow, target T, extractor func(StructAElement) T) map[int64]struct{} {
ids := make(map[int64]struct{})
for _, row := range data {
for _, e := range row.StructA {
if extractor(e) == target {
ids[row.ID] = struct{}{}
break
}
}
}
return ids
}
// GtArrayContainsAll returns IDs whose StructA contains every value in `targets` (each via
// extractor on at least one element).
func GtArrayContainsAll[T comparable](data []StructARow, targets []T, extractor func(StructAElement) T) map[int64]struct{} {
ids := make(map[int64]struct{})
for _, row := range data {
seen := make(map[T]bool, len(targets))
for _, e := range row.StructA {
seen[extractor(e)] = true
}
all := true
for _, t := range targets {
if !seen[t] {
all = false
break
}
}
if all {
ids[row.ID] = struct{}{}
}
}
return ids
}
// GtArrayContainsAny returns IDs whose StructA contains any of the targets.
func GtArrayContainsAny[T comparable](data []StructARow, targets []T, extractor func(StructAElement) T) map[int64]struct{} {
ids := make(map[int64]struct{})
want := make(map[T]bool, len(targets))
for _, t := range targets {
want[t] = true
}
for _, row := range data {
for _, e := range row.StructA {
if want[extractor(e)] {
ids[row.ID] = struct{}{}
break
}
}
}
return ids
}
// L2Distance returns the squared L2 distance between two equal-length float32 vectors.
func L2Distance(a, b []float32) float64 {
var s float64
for i := range a {
d := float64(a[i] - b[i])
s += d * d
}
return s
}
// GtElementSearchNoFilter returns the top-K (rowID, bestScore) pairs for an element-level vector
// search with no filter. Each row contributes its best matching element's score (max for COSINE/IP,
// min for L2). Mirrors python `gt_element_search_no_filter`.
func GtElementSearchNoFilter(data []StructARow, queryVector []float32, metric string, limit int) []int64 {
type rowScore struct {
id int64
score float64
}
descending := metric == "COSINE" || metric == "IP"
scores := make([]rowScore, 0, len(data))
for _, row := range data {
var best float64
hasBest := false
for _, e := range row.StructA {
s := scoreFor(queryVector, e.Embedding, metric)
if !hasBest || (descending && s > best) || (!descending && s < best) {
best = s
hasBest = true
}
}
if hasBest {
scores = append(scores, rowScore{row.ID, best})
}
}
// stable sort by score
for i := 1; i < len(scores); i++ {
j := i
for j > 0 {
lhs := scores[j-1].score
rhs := scores[j].score
if (descending && lhs >= rhs) || (!descending && lhs <= rhs) {
break
}
scores[j-1], scores[j] = scores[j], scores[j-1]
j--
}
}
if limit > len(scores) {
limit = len(scores)
}
out := make([]int64, limit)
for i := 0; i < limit; i++ {
out[i] = scores[i].id
}
return out
}
func scoreFor(q, v []float32, metric string) float64 {
switch metric {
case "COSINE":
return float64(CosineSimilarity(q, v))
case "L2":
return L2Distance(q, v)
case "IP":
var s float64
for i := range q {
s += float64(q[i]) * float64(v[i])
}
return s
}
return 0
}
// IDSetToSorted is a tiny utility to turn the ID maps into deterministic int64 slices for diff
// printing in failed assertions.
func IDSetToSorted(set map[int64]struct{}) []int64 {
out := make([]int64, 0, len(set))
for id := range set {
out = append(out, id)
}
for i := 1; i < len(out); i++ {
j := i
for j > 0 && out[j-1] > out[j] {
out[j-1], out[j] = out[j], out[j-1]
j--
}
}
return out
}
tests/go_client/testcases/helper/struct_array_helper.go
+178
View File
@@ -0,0 +1,178 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helper
import (
"fmt"
"math/rand"
"github.com/milvus-io/milvus/client/v2/entity"
)
// Mirrored constants from tests/python_client/milvus_client/test_milvus_client_struct_array.py
const (
StructArrayPrefix = "struct_array"
StructArrayDefaultDim = 128
StructArrayDefaultCapacity = 100
// default_nb in python_client/common/common_type.py is 3000; we stick to a smaller number
// for Go SDK tests so a single test stays under ~5s while still exercising the code paths.
StructArrayDefaultNb = 500
)
// StructArraySchemaOption lets tests customize the canonical struct-array schema.
type StructArraySchemaOption struct {
Dim int
Capacity int
IncludeClipStr bool // add clip_str sub-field (default true)
IncludeEmb1 bool // add clip_embedding1 sub-field (default true)
IncludeEmb2 bool // add clip_embedding2 sub-field (default true)
CollectionName string
NormalMaxLength int64 // max_length for clip_str; defaults to 65535
}
// DefaultStructArraySchemaOption returns the canonical struct array schema option matching the
// Python tests: id (Int64 PK), normal_vector (FloatVector), clips (struct array with
// clip_str + clip_embedding1 + clip_embedding2).
func DefaultStructArraySchemaOption(name string) StructArraySchemaOption {
return StructArraySchemaOption{
Dim: StructArrayDefaultDim,
Capacity: StructArrayDefaultCapacity,
IncludeClipStr: true,
IncludeEmb1: true,
IncludeEmb2: true,
CollectionName: name,
NormalMaxLength: 65535,
}
}
// CreateStructArraySchema builds the canonical struct-array schema used across the Python tests.
// The returned schema and the returned StructSchema are paired - the StructSchema must be passed
// to WithStructArrayColumn when inserting struct array data.
func CreateStructArraySchema(opt StructArraySchemaOption) (*entity.Schema, *entity.StructSchema) {
if opt.Dim == 0 {
opt.Dim = StructArrayDefaultDim
}
if opt.Capacity == 0 {
opt.Capacity = StructArrayDefaultCapacity
}
if opt.NormalMaxLength == 0 {
opt.NormalMaxLength = 65535
}
structSchema := entity.NewStructSchema()
if opt.IncludeClipStr {
structSchema.WithField(entity.NewField().WithName("clip_str").
WithDataType(entity.FieldTypeVarChar).WithMaxLength(opt.NormalMaxLength))
}
if opt.IncludeEmb1 {
structSchema.WithField(entity.NewField().WithName("clip_embedding1").
WithDataType(entity.FieldTypeFloatVector).WithDim(int64(opt.Dim)))
}
if opt.IncludeEmb2 {
structSchema.WithField(entity.NewField().WithName("clip_embedding2").
WithDataType(entity.FieldTypeFloatVector).WithDim(int64(opt.Dim)))
}
schema := entity.NewSchema().WithName(opt.CollectionName).
WithField(entity.NewField().WithName("id").
WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("normal_vector").
WithDataType(entity.FieldTypeFloatVector).WithDim(int64(opt.Dim))).
WithField(entity.NewField().WithName("clips").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(int64(opt.Capacity)).
WithStructSchema(structSchema))
return schema, structSchema
}
// StructArrayTestData carries the generated columns ready to feed into
// WithInt64Column / WithFloatVectorColumn / WithStructArrayColumn.
type StructArrayTestData struct {
IDs []int64
NormalVectors [][]float32
ClipsRows []map[string]any
Dim int
}
// GenerateStructArrayData generates numRows rows matching the Python generator:
// - id: sequential int64
// - normal_vector: random float vector of `dim`
// - clips: array of 1..min(capacity,20) struct elements with clip_str + clip_embedding1/2
//
// Pass includeClipStr/IncludeEmb1/IncludeEmb2 = false to omit the corresponding sub-field.
func GenerateStructArrayData(numRows int, opt StructArraySchemaOption) StructArrayTestData {
if opt.Dim == 0 {
opt.Dim = StructArrayDefaultDim
}
if opt.Capacity == 0 {
opt.Capacity = StructArrayDefaultCapacity
}
maxLen := opt.Capacity
if maxLen > 20 {
maxLen = 20
}
data := StructArrayTestData{
IDs: make([]int64, numRows),
NormalVectors: make([][]float32, numRows),
ClipsRows: make([]map[string]any, numRows),
Dim: opt.Dim,
}
for i := 0; i < numRows; i++ {
data.IDs[i] = int64(i)
data.NormalVectors[i] = RandFloatVector(opt.Dim)
arrLen := 1 + rand.Intn(maxLen)
strs := make([]string, 0, arrLen)
emb1 := make([][]float32, 0, arrLen)
emb2 := make([][]float32, 0, arrLen)
for j := 0; j < arrLen; j++ {
strs = append(strs, fmtItem(i, j))
emb1 = append(emb1, RandFloatVector(opt.Dim))
emb2 = append(emb2, RandFloatVector(opt.Dim))
}
row := map[string]any{}
if opt.IncludeClipStr {
row["clip_str"] = strs
}
if opt.IncludeEmb1 {
row["clip_embedding1"] = emb1
}
if opt.IncludeEmb2 {
row["clip_embedding2"] = emb2
}
data.ClipsRows[i] = row
}
return data
}
// RandFloatVector returns a random float32 vector of the given dimension.
func RandFloatVector(dim int) []float32 {
v := make([]float32, dim)
for i := range v {
v[i] = rand.Float32()
}
return v
}
func fmtItem(i, j int) string {
// "item_{i}_{j}" — match Python generator for diagnostic parity.
return fmt.Sprintf("item_%d_%d", i, j)
}
tests/go_client/testcases/struct_array_element_query_test.go
+691
View File
@@ -0,0 +1,691 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// L0 ports of tests/python_client/milvus_client/test_milvus_client_struct_array_element_query.py.
// Where Python uses class-level fixtures we collapse subtests into a single Go test function via
// t.Run so the collection setup amortizes across cases.
package testcases
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/client/v2/index"
client "github.com/milvus-io/milvus/client/v2/milvusclient"
"github.com/milvus-io/milvus/tests/go_client/common"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
)
const (
elemQuerySealedNb = hp.StructAElemSealedNb
elemQueryGrowingNb = hp.StructAElemGrowingNb
)
// setupElemSharedCollection mirrors the python class fixture: create canonical schema, build
// indexes on normal_vector and structA[embedding], insert sealedNb sealed + growingNb growing
// rows, load. Returns coll name + struct schema + concatenated dataset for ground truth.
func setupElemSharedCollection(t *testing.T, ctx CtxT, mc MC, namePrefix string, opt hp.StructAElementSchemaOption) (string, *entity.StructSchema, []hp.StructARow) {
collName := common.GenRandomString(namePrefix, 6)
opt.CollectionName = collName
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
// Sealed batch
sealed := hp.GenerateStructAElementData(elemQuerySealedNb, 0, opt)
insertElemDataset(t, ctx, mc, collName, structSchema, sealed, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
// Growing batch (no flush)
growing := hp.GenerateStructAElementData(elemQueryGrowingNb, int64(elemQuerySealedNb), opt)
insertElemDataset(t, ctx, mc, collName, structSchema, growing, opt)
// Indexes + load
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
all := append([]hp.StructARow{}, sealed.Rows...)
all = append(all, growing.Rows...)
return collName, structSchema, all
}
func insertElemDataset(t *testing.T, ctx CtxT, mc MC, collName string, structSchema *entity.StructSchema, ds hp.StructAElementDataset, opt hp.StructAElementSchemaOption) {
ids, vectors, docInts, docVChars, structRows := ds.ToInsertColumns()
insertOpt := client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", ids)
if opt.IncludeDocInt {
insertOpt = insertOpt.WithInt64Column("doc_int", docInts)
}
if opt.IncludeDocVChar {
insertOpt = insertOpt.WithVarcharColumn("doc_varchar", docVChars)
}
insertOpt = insertOpt.
WithFloatVectorColumn("normal_vector", opt.Dim, vectors).
WithStructArrayColumn("structA", structSchema, structRows)
_, err := mc.Insert(ctx, insertOpt)
common.CheckErr(t, err, true)
}
// queryAllIDs runs Query and returns sorted IDs from the "id" output column.
func queryAllIDs(t *testing.T, ctx CtxT, mc MC, collName, expr string, limit int) []int64 {
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter(expr).WithOutputFields("id").WithLimit(limit).
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
col := rs.GetColumn("id")
require.NotNil(t, col, "id column missing")
out := make([]int64, rs.ResultCount)
for i := 0; i < rs.ResultCount; i++ {
v, err := col.Get(i)
require.NoError(t, err)
out[i] = v.(int64)
}
return sortInt64s(out)
}
func sortInt64s(s []int64) []int64 {
for i := 1; i < len(s); i++ {
j := i
for j > 0 && s[j-1] > s[j] {
s[j-1], s[j] = s[j], s[j-1]
j--
}
}
return s
}
func subset(a, b []int64) bool {
bs := make(map[int64]struct{}, len(b))
for _, x := range b {
bs[x] = struct{}{}
}
for _, x := range a {
if _, ok := bs[x]; !ok {
return false
}
}
return true
}
// =============================================================================
// 1. TestMilvusClientStructArrayElementContains (4 L0)
// =============================================================================
func TestStructArrayElementContains(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
opt := hp.DefaultStructAElementSchemaOption("")
collName, _, data := setupElemSharedCollection(t, ctx, mc, hp.StructAElemPrefix+"_ac_shared", opt)
t.Run("array_contains_int_subfield", func(t *testing.T) {
const target int64 = 100 // row 1 elem 0 → int_val = 1*100+0 = 100
ids := queryAllIDs(t, ctx, mc, collName, "array_contains(structA[int_val], 100)", 50)
gt := hp.GtArrayContains(data, target, func(e hp.StructAElement) int64 { return e.IntVal })
require.True(t, subset(ids, hp.IDSetToSorted(gt)),
"server returned IDs not in ground truth: got %v gt %v", ids, hp.IDSetToSorted(gt))
require.Greater(t, len(ids), 0)
})
t.Run("array_contains_varchar_subfield", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName, `array_contains(structA[color], "Red")`, 50)
require.Greater(t, len(ids), 0)
// Every returned row must contain at least one Red element.
expected := hp.GtArrayContains(data, "Red", func(e hp.StructAElement) string { return e.Color })
require.True(t, subset(ids, hp.IDSetToSorted(expected)))
})
t.Run("array_contains_all_struct_subfield", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName, `array_contains_all(structA[color], ["Red", "Blue"])`, 50)
require.Greater(t, len(ids), 0)
gt := hp.GtArrayContainsAll(data, []string{"Red", "Blue"}, func(e hp.StructAElement) string { return e.Color })
require.True(t, subset(ids, hp.IDSetToSorted(gt)))
})
t.Run("array_contains_any_struct_subfield", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName, `array_contains_any(structA[category], ["A", "B"])`, 50)
require.Greater(t, len(ids), 0)
gt := hp.GtArrayContainsAny(data, []string{"A", "B"}, func(e hp.StructAElement) string { return e.Category })
require.True(t, subset(ids, hp.IDSetToSorted(gt)))
})
}
// =============================================================================
// 2. TestMilvusClientStructArrayElementQuery (3 L0)
// =============================================================================
func TestStructArrayElementQueryBasics(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
opt := hp.DefaultStructAElementSchemaOption("")
collName, _, data := setupElemSharedCollection(t, ctx, mc, hp.StructAElemPrefix+"_efq_shared", opt)
t.Run("element_filter_query_basic", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName, `element_filter(structA, $[int_val] > 200)`, 50)
require.Greater(t, len(ids), 0)
gt := hp.GtElementFilter(data, func(e hp.StructAElement) bool { return e.IntVal > 200 }, nil)
require.True(t, subset(ids, hp.IDSetToSorted(gt)),
"got %v not subset of gt %v", ids, hp.IDSetToSorted(gt))
})
t.Run("element_filter_query_compound", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName,
`element_filter(structA, $[color] == "Red" && $[int_val] > 100)`, 50)
require.Greater(t, len(ids), 0)
gt := hp.GtElementFilter(data,
func(e hp.StructAElement) bool { return e.Color == "Red" && e.IntVal > 100 }, nil)
require.True(t, subset(ids, hp.IDSetToSorted(gt)))
})
t.Run("element_filter_query_with_doc_filter", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName,
`doc_int > 100 && element_filter(structA, $[color] == "Red")`, 50)
require.Greater(t, len(ids), 0)
gt := hp.GtElementFilter(data,
func(e hp.StructAElement) bool { return e.Color == "Red" },
func(r hp.StructARow) bool { return r.DocInt > 100 })
require.True(t, subset(ids, hp.IDSetToSorted(gt)))
})
}
// =============================================================================
// 4. TestMilvusClientStructArrayElementSTLSortIndex (2 L0)
// =============================================================================
// runSTLSortCase creates a fresh collection with HNSW(normal_vector + structA[embedding]) and
// STL_SORT(structA[<scalarField>]), inserts sealed+growing data, loads, and asserts a basic
// query returns the expected number of rows. Mirrors stl_sort_index_create_int / _varchar.
func runSTLSortCase(t *testing.T, ctx CtxT, mc MC, scalarField, namePrefix string) {
collName := common.GenRandomString(namePrefix, 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
schema, structSchema := hp.CreateStructAElementSchema(opt)
// Pre-create with index params to exercise the same code path as the python prepare_index_params.
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
sealed := hp.GenerateStructAElementData(elemQuerySealedNb, 0, opt)
insertElemDataset(t, ctx, mc, collName, structSchema, sealed, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
growing := hp.GenerateStructAElementData(elemQueryGrowingNb, int64(elemQuerySealedNb), opt)
insertElemDataset(t, ctx, mc, collName, structSchema, growing, opt)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA["+scalarField+"]",
index.NewGenericIndex("stl_sort_idx", map[string]string{"index_type": "STL_SORT"})))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("id < 10").WithOutputFields("id").WithLimit(10).
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 10, rs.ResultCount)
}
func TestStructArrayElementSTLSortIndexCreateInt(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
runSTLSortCase(t, ctx, mc, "int_val", hp.StructAElemPrefix+"_stl_int")
}
func TestStructArrayElementSTLSortIndexCreateVarchar(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
runSTLSortCase(t, ctx, mc, "str_val", hp.StructAElemPrefix+"_stl_vc")
}
// =============================================================================
// 5. TestMilvusClientStructArrayElementIndexRegression (2 L0)
// =============================================================================
// runInvertedIndexCase creates a collection with an INVERTED index on a struct sub-field, then
// runs a search filtered by element_filter on the same sub-field — verifying the index was
// actually built with the right inverted-index code path (PR #48183 regression).
func runInvertedIndexCase(t *testing.T, ctx CtxT, mc MC, scalarField, namePrefix, filter string, gtFn func(hp.StructAElement) bool) {
collName := common.GenRandomString(namePrefix, 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeDocInt = false
opt.IncludeDocVChar = false
opt.IncludeFloatVal = false
opt.IncludeCategory = false
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
sealed := hp.GenerateStructAElementData(elemQuerySealedNb, 0, opt)
insertElemDataset(t, ctx, mc, collName, structSchema, sealed, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA["+scalarField+"]",
index.NewGenericIndex("inv_idx", map[string]string{"index_type": "INVERTED"})))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
// Search using row 0's first embedding (deterministic from SeedVector).
queryEmb := entity.FloatVectorArray{entity.FloatVector(hp.SeedVector(0, opt.Dim))}
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10, []entity.Vector{queryEmb}).
WithANNSField("structA[embedding]").
WithFilter(filter).
WithOutputFields("id").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.GreaterOrEqual(t, len(rs), 1)
require.Greater(t, rs[0].ResultCount, 0)
// Sanity: every returned ID must satisfy the ground-truth predicate.
gt := hp.GtElementFilter(sealed.Rows, gtFn, nil)
idCol := rs[0].GetColumn("id")
for i := 0; i < rs[0].ResultCount; i++ {
v, err := idCol.Get(i)
require.NoError(t, err)
_, ok := gt[v.(int64)]
require.True(t, ok, "row %d not in ground truth set", v.(int64))
}
}
func TestStructArrayElementInvertedIndexVarchar(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
runInvertedIndexCase(t, ctx, mc, "color", hp.StructAElemPrefix+"_inv_vc",
`element_filter(structA, $[color] == "Red")`,
func(e hp.StructAElement) bool { return e.Color == "Red" })
}
func TestStructArrayElementInvertedIndexInt(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
runInvertedIndexCase(t, ctx, mc, "int_val", hp.StructAElemPrefix+"_inv_int",
`element_filter(structA, $[int_val] > 100)`,
func(e hp.StructAElement) bool { return e.IntVal > 100 })
}
// =============================================================================
// 6. TestMilvusClientStructArrayElementQueryCorrectness (10 L0)
// =============================================================================
// setupCorrectnessCollection mirrors python `_setup_with_controlled_data`: inserts background
// inert rows then the controlled test rows. We shrink background rows from 3500 → 100 inert to
// keep tests fast while still exercising sealed+growing.
func setupCorrectnessCollection(t *testing.T, ctx CtxT, mc MC, namePrefix string, controlled []hp.StructARow) (string, *entity.StructSchema, []hp.StructARow) {
collName := common.GenRandomString(namePrefix, 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeDocVChar = false
opt.IncludeFloatVal = false
opt.IncludeCategory = false
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
bgRows := make([]hp.StructARow, 0, 100)
bgStart := int64(100000)
for i := int64(0); i < 100; i++ {
bgRows = append(bgRows, hp.MakeInertRow(bgStart+i, opt))
}
insertCustomRows(t, ctx, mc, collName, structSchema, bgRows, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
insertCustomRows(t, ctx, mc, collName, structSchema, controlled, opt)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
all := append([]hp.StructARow{}, bgRows...)
all = append(all, controlled...)
return collName, structSchema, all
}
func insertCustomRows(t *testing.T, ctx CtxT, mc MC, collName string, structSchema *entity.StructSchema, rows []hp.StructARow, opt hp.StructAElementSchemaOption) {
if len(rows) == 0 {
return
}
ids, vectors, docInts, docVChars, structRows := hp.RowsToColumns(rows, opt)
insertOpt := client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", ids)
if opt.IncludeDocInt {
insertOpt = insertOpt.WithInt64Column("doc_int", docInts)
}
if opt.IncludeDocVChar {
insertOpt = insertOpt.WithVarcharColumn("doc_varchar", docVChars)
}
insertOpt = insertOpt.
WithFloatVectorColumn("normal_vector", opt.Dim, vectors).
WithStructArrayColumn("structA", structSchema, structRows)
_, err := mc.Insert(ctx, insertOpt)
common.CheckErr(t, err, true)
}
func TestStructArrayElementQueryCorrectness(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
t.Run("element_filter_query_exact_ids", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := []hp.StructARow{
hp.MakeRow(0, opt, []hp.StructAElement{{IntVal: 10}, {IntVal: 20}, {IntVal: 30}}),
hp.MakeRow(1, opt, []hp.StructAElement{{IntVal: 3}, {IntVal: 4}}),
hp.MakeRow(2, opt, []hp.StructAElement{{IntVal: 1}, {IntVal: 100}}),
hp.MakeRow(3, opt, []hp.StructAElement{{IntVal: 6}}),
hp.MakeRow(4, opt, []hp.StructAElement{{IntVal: 5}}),
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_exact_ids", controlled)
ids := queryAllIDs(t, ctx, mc, collName, `element_filter(structA, $[int_val] > 5)`, 100)
require.Equal(t, []int64{0, 2, 3}, ids)
})
t.Run("element_filter_query_no_match", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := []hp.StructARow{
hp.MakeRow(0, opt, []hp.StructAElement{{IntVal: 1}, {IntVal: 2}}),
hp.MakeRow(1, opt, []hp.StructAElement{{IntVal: 3}}),
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_no_match", controlled)
ids := queryAllIDs(t, ctx, mc, collName, `element_filter(structA, $[int_val] > 9999)`, 100)
require.Empty(t, ids)
})
t.Run("element_filter_query_offset_correctness", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := make([]hp.StructARow, 0, 20)
for i := int64(0); i < 20; i++ {
controlled = append(controlled, hp.MakeRow(i, opt, []hp.StructAElement{{IntVal: i * 10}}))
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_offset", controlled)
expr := `element_filter(structA, $[int_val] > 50)`
all := queryAllIDs(t, ctx, mc, collName, expr, 100)
page1 := queryPage(t, ctx, mc, collName, expr, 5, 0)
page2 := queryPage(t, ctx, mc, collName, expr, 5, 5)
page3 := queryPage(t, ctx, mc, collName, expr, 5, 10)
// Pages must be disjoint
require.Empty(t, intersect(page1, page2))
require.Empty(t, intersect(page1, page3))
require.Empty(t, intersect(page2, page3))
// And union ⊆ all
union := append(append(append([]int64{}, page1...), page2...), page3...)
require.True(t, subset(union, all),
"union of pages %v not subset of full result %v", union, all)
})
t.Run("element_filter_query_returned_elements_correctness", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := []hp.StructARow{
hp.MakeRow(0, opt, []hp.StructAElement{
{IntVal: 10, Color: "Red"},
{IntVal: 20, Color: "Blue"},
{IntVal: 30, Color: "Green"},
}),
hp.MakeRow(1, opt, []hp.StructAElement{
{IntVal: 5, Color: "Red"},
}),
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_elem_correct", controlled)
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter(`element_filter(structA, $[int_val] > 15)`).
WithOutputFields("id", "structA").WithLimit(100).
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 1, rs.ResultCount)
require.EqualValues(t, int64(0), mustInt64(t, rs.GetColumn("id"), 0))
structVal, err := rs.GetColumn("structA").Get(0)
require.NoError(t, err)
m := structVal.(map[string]any)
intVals := m["int_val"].([]int64)
require.GreaterOrEqual(t, len(intVals), 1)
})
t.Run("element_filter_count_exact", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := make([]hp.StructARow, 10)
for i := int64(0); i < 10; i++ {
val := int64(1)
if i >= 5 {
val = 100
}
controlled[i] = hp.MakeRow(i, opt, []hp.StructAElement{{IntVal: val}})
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_count", controlled)
require.EqualValues(t, 5, queryCountStar(t, ctx, mc, collName,
`element_filter(structA, $[int_val] > 50)`))
require.EqualValues(t, 10, queryCountStar(t, ctx, mc, collName,
`element_filter(structA, $[int_val] > 0)`))
})
t.Run("element_filter_query_same_element_semantic", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := []hp.StructARow{
hp.MakeRow(0, opt, []hp.StructAElement{
{IntVal: 10, Color: "Red"},
{IntVal: 20, Color: "Blue"},
}),
hp.MakeRow(1, opt, []hp.StructAElement{{IntVal: 20, Color: "Red"}}),
hp.MakeRow(2, opt, []hp.StructAElement{{IntVal: 5, Color: "Green"}}),
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_same_elem", controlled)
ids := queryAllIDs(t, ctx, mc, collName,
`element_filter(structA, $[color] == "Red" && $[int_val] > 15)`, 100)
require.Equal(t, []int64{1}, ids, "same-element semantic: only row 1 has Red AND int_val>15 in same elem")
})
t.Run("match_all_query_exact_verification", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := []hp.StructARow{
hp.MakeRow(0, opt, []hp.StructAElement{{IntVal: 11}, {IntVal: 20}, {IntVal: 30}}),
hp.MakeRow(1, opt, []hp.StructAElement{{IntVal: 5}, {IntVal: 20}}),
hp.MakeRow(2, opt, []hp.StructAElement{{IntVal: 100}}),
hp.MakeRow(3, opt, []hp.StructAElement{{IntVal: 1}, {IntVal: 2}, {IntVal: 3}}),
hp.MakeRow(4, opt, []hp.StructAElement{{IntVal: 10}, {IntVal: 15}}),
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_mall_exact", controlled)
ids := queryAllIDs(t, ctx, mc, collName, `MATCH_ALL(structA, $[int_val] > 10)`, 100)
require.Equal(t, []int64{0, 2}, ids)
})
t.Run("match_exact_query_verification", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := []hp.StructARow{
hp.MakeRow(0, opt, []hp.StructAElement{
{IntVal: 1, Color: "Red"}, {IntVal: 2, Color: "Red"}, {IntVal: 3, Color: "Red"},
}),
hp.MakeRow(1, opt, []hp.StructAElement{
{IntVal: 1, Color: "Red"}, {IntVal: 2, Color: "Blue"}, {IntVal: 3, Color: "Red"},
}),
hp.MakeRow(2, opt, []hp.StructAElement{
{IntVal: 1, Color: "Red"}, {IntVal: 2, Color: "Blue"},
}),
hp.MakeRow(3, opt, []hp.StructAElement{{IntVal: 1, Color: "Blue"}}),
hp.MakeRow(4, opt, []hp.StructAElement{
{IntVal: 1, Color: "Red"}, {IntVal: 2, Color: "Red"},
}),
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_mexact", controlled)
ids := queryAllIDs(t, ctx, mc, collName,
`MATCH_EXACT(structA, $[color] == "Red", threshold=2)`, 100)
require.Equal(t, []int64{1, 4}, ids)
})
t.Run("array_contains_exact_result_set", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := []hp.StructARow{
hp.MakeRow(0, opt, []hp.StructAElement{{IntVal: 1, Color: "Red"}}),
hp.MakeRow(1, opt, []hp.StructAElement{{IntVal: 2, Color: "Blue"}}),
hp.MakeRow(2, opt, []hp.StructAElement{
{IntVal: 3, Color: "Red"}, {IntVal: 4, Color: "Blue"},
}),
hp.MakeRow(3, opt, []hp.StructAElement{{IntVal: 5, Color: "Green"}}),
}
collName, _, _ := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_ac_exact", controlled)
ids := queryAllIDs(t, ctx, mc, collName, `array_contains(structA[color], "Red")`, 100)
require.Equal(t, []int64{0, 2}, ids)
ids2 := queryAllIDs(t, ctx, mc, collName, `array_contains_all(structA[color], ["Red", "Blue"])`, 100)
require.Equal(t, []int64{2}, ids2)
})
t.Run("element_filter_query_no_false_positives", func(t *testing.T) {
opt := hp.DefaultStructAElementSchemaOption("")
controlled := make([]hp.StructARow, 0, 50)
for i := int64(0); i < 50; i++ {
r := uint(i)
numElems := 2 + int(r%5) // 2..6
elems := make([]hp.StructAElement, numElems)
for j := 0; j < numElems; j++ {
elems[j] = hp.StructAElement{
IntVal: i*100 + int64(j),
Color: hp.StructAElemColors[j%3],
}
}
controlled = append(controlled, hp.MakeRow(i, opt, elems))
}
collName, _, allRows := setupCorrectnessCollection(t, ctx, mc, hp.StructAElemPrefix+"_no_fp", controlled)
expr := `element_filter(structA, $[color] == "Blue" && $[int_val] > 2000)`
ids := queryAllIDs(t, ctx, mc, collName, expr, 100)
gt := hp.GtElementFilter(allRows,
func(e hp.StructAElement) bool { return e.Color == "Blue" && e.IntVal > 2000 }, nil)
require.Equal(t, hp.IDSetToSorted(gt), ids,
"server returned IDs differ from ground truth")
})
}
func mustInt64(t *testing.T, c interface{ Get(int) (any, error) }, idx int) int64 {
v, err := c.Get(idx)
require.NoError(t, err)
return v.(int64)
}
func queryPage(t *testing.T, ctx CtxT, mc MC, collName, expr string, limit, offset int) []int64 {
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter(expr).WithOutputFields("id").WithLimit(limit).WithOffset(offset).
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
col := rs.GetColumn("id")
out := make([]int64, rs.ResultCount)
for i := 0; i < rs.ResultCount; i++ {
v, _ := col.Get(i)
out[i] = v.(int64)
}
return out
}
func queryCountStar(t *testing.T, ctx CtxT, mc MC, collName, expr string) int64 {
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter(expr).WithOutputFields("count(*)").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 1, rs.ResultCount)
v, err := rs.GetColumn("count(*)").Get(0)
require.NoError(t, err)
return v.(int64)
}
func intersect(a, b []int64) []int64 {
bm := make(map[int64]struct{}, len(b))
for _, x := range b {
bm[x] = struct{}{}
}
out := []int64{}
for _, x := range a {
if _, ok := bm[x]; ok {
out = append(out, x)
}
}
return out
}
// =============================================================================
// 8. TestMilvusClientStructArrayElementMatchQuery (2 L0)
// =============================================================================
func TestStructArrayElementMatchQuery(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(hp.StructAElemPrefix+"_mq", 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeDocVChar = false
opt.IncludeFloatVal = false
opt.IncludeCategory = false
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
ds := hp.GenerateStructAElementData(elemQuerySealedNb, 0, opt)
insertElemDataset(t, ctx, mc, collName, structSchema, ds, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
t.Run("match_query_all", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName, `MATCH_ALL(structA, $[int_val] > 0)`, 100)
gt := hp.GtMatch(ds.Rows, "MATCH_ALL", func(e hp.StructAElement) bool { return e.IntVal > 0 }, 0, nil)
require.True(t, subset(ids, hp.IDSetToSorted(gt)),
"got %v not subset of gt %v", ids, hp.IDSetToSorted(gt))
})
t.Run("match_query_any", func(t *testing.T) {
ids := queryAllIDs(t, ctx, mc, collName, `MATCH_ANY(structA, $[str_val] == "row_10_elem_0")`, 100)
gt := hp.GtMatch(ds.Rows, "MATCH_ANY",
func(e hp.StructAElement) bool { return e.StrVal == "row_10_elem_0" }, 0, nil)
require.Equal(t, hp.IDSetToSorted(gt), ids)
})
}
tests/go_client/testcases/struct_array_element_search_test.go
+744
View File
@@ -0,0 +1,744 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// L0 ports of tests/python_client/milvus_client/
// test_milvus_client_struct_array_element_search.py.
//
// Three Python tests are explicitly @pytest.mark.xfail and we mirror that with t.Skip:
// - test_element_filter_search_basic_cosine (flaky element_indices on growing segment)
// - test_element_filter_search_basic_l2 (same root cause)
// - test_element_filter_search_verify_in_struct_offset (pymilvus element_indices not exposed)
package testcases
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus/client/v2/column"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/client/v2/index"
client "github.com/milvus-io/milvus/client/v2/milvusclient"
"github.com/milvus-io/milvus/tests/go_client/common"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
)
const elemSearchPrefix = "struct_elem_search"
// =============================================================================
// 1. TestMilvusClientStructArrayElementFilterSearch (5 L0, 3 skipped as xfail)
// =============================================================================
func TestStructArrayElementFilterSearchBasicCosine(t *testing.T) {
t.Skip("xfail in python: flaky element-level search on growing segment returns wrong element-to-row mapping")
}
func TestStructArrayElementFilterSearchBasicL2(t *testing.T) {
t.Skip("xfail in python: same flaky element-to-row mapping issue")
}
func TestStructArrayElementFilterSearchVerifyInStructOffset(t *testing.T) {
t.Skip("xfail in python: element_indices not yet re-exposed after PR #3240 refactoring")
}
// TestStructArrayElementFilterSearchWithDocLevelFilter ports
// test_element_filter_search_with_doc_level_filter.
func TestStructArrayElementFilterSearchWithDocLevelFilter(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(elemSearchPrefix+"_ef_doc", 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeSize = true
opt.IncludeCategory = false
opt.IncludeFloatVal = true
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
// 500 rows is enough to validate the doc filter without bloating runtime.
ds := hp.GenerateStructAElementData(500, 0, opt)
insertElemDataset(t, ctx, mc, collName, structSchema, ds, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
indexAndLoadElem(t, ctx, mc, collName)
// Use row 200's first element embedding as query; filter pins doc_int>100 + str_val match.
// Single-vector search (not EmbList) — element_filter+vector search on struct sub-vector
// works with regular FloatVector when only one query vector is involved.
queryVec := ds.Rows[200].StructA[0].Embedding
// Plain-vector search against an EmbList-indexed field must override metric_type to the
// underlying COSINE (the index's MAX_SIM_COSINE is reserved for embedding-list queries).
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10, []entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("structA[embedding]").
WithSearchParam("metric_type", "COSINE").
WithFilter(`doc_int > 100 && element_filter(structA, $[str_val] == "row_200_elem_0")`).
WithOutputFields("id", "doc_int").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.GreaterOrEqual(t, len(rs), 1)
require.Greater(t, rs[0].ResultCount, 0)
// Top-1 must be row 200 (queried its own vector).
idCol := rs[0].GetColumn("id")
docCol := rs[0].GetColumn("doc_int")
for i := 0; i < rs[0].ResultCount; i++ {
v, _ := docCol.Get(i)
require.Greater(t, v.(int64), int64(100))
}
first, _ := idCol.Get(0)
require.EqualValues(t, int64(200), first.(int64))
}
// TestStructArrayElementFilterSearchCompoundSameElementSemantic ports
// test_element_filter_search_compound_same_element_semantic.
func TestStructArrayElementFilterSearchCompoundSameElementSemantic(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(elemSearchPrefix+"_ef_semantic", 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeSize = true
opt.IncludeCategory = false
opt.IncludeFloatVal = true
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
targetVec := hp.SeedVector(77777, opt.Dim)
rows := []hp.StructARow{
{
ID: 0, DocInt: 0, DocVarChar: "cat_0",
NormalVector: hp.SeedVector(99990, opt.Dim),
StructA: []hp.StructAElement{
{Embedding: hp.SeedVector(0, opt.Dim), IntVal: 1, StrVal: "a", FloatVal: 0.1, Color: "Red", Size: "S"},
{Embedding: targetVec, IntVal: 2, StrVal: "b", FloatVal: 0.2, Color: "Blue", Size: "L"},
},
},
{
ID: 1, DocInt: 1, DocVarChar: "cat_1",
NormalVector: hp.SeedVector(99991, opt.Dim),
StructA: []hp.StructAElement{
{Embedding: targetVec, IntVal: 10, StrVal: "x", FloatVal: 1.0, Color: "Red", Size: "L"},
},
},
{
ID: 2, DocInt: 2, DocVarChar: "cat_2",
NormalVector: hp.SeedVector(99992, opt.Dim),
StructA: []hp.StructAElement{
{Embedding: hp.SeedVector(20, opt.Dim), IntVal: 20, StrVal: "p", FloatVal: 2.0, Color: "Blue", Size: "S"},
},
},
}
insertCustomRows(t, ctx, mc, collName, structSchema, rows, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
indexAndLoadElem(t, ctx, mc, collName)
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10, []entity.Vector{entity.FloatVector(targetVec)}).
WithANNSField("structA[embedding]").
WithSearchParam("metric_type", "COSINE").
WithFilter(`element_filter(structA, $[color] == "Red" && $[size] == "L")`).
WithOutputFields("id").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
matched := map[int64]bool{}
idCol := rs[0].GetColumn("id")
for i := 0; i < rs[0].ResultCount; i++ {
v, _ := idCol.Get(i)
matched[v.(int64)] = true
}
require.False(t, matched[0], "row 0: Red and L are on different elements; must NOT match")
require.True(t, matched[1], "row 1: elem[0]={Red,L} satisfies same-element semantic")
}
// indexAndLoadElem builds the canonical 2 indexes for plain-vector struct sub-search.
// Use entity.COSINE on the sub-vector (NOT MaxSimCosine) so plain FloatVector searches work.
// Tests that need EmbList/MaxSim semantics build their own indexes.
func indexAndLoadElem(t *testing.T, ctx CtxT, mc MC, collName string) {
_, err := mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
}
// =============================================================================
// 2. TestMilvusClientStructArrayElementMatchSearch (4 L0)
// =============================================================================
// matchSearchSetup creates the canonical match-search collection (no doc_varchar, has size).
func matchSearchSetup(t *testing.T, ctx CtxT, mc MC, namePrefix string, rows []hp.StructARow) (string, hp.StructAElementSchemaOption) {
collName := common.GenRandomString(namePrefix, 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeDocVChar = false
opt.IncludeCategory = false
opt.IncludeSize = true
opt.IncludeFloatVal = true
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
insertCustomRows(t, ctx, mc, collName, structSchema, rows, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
indexAndLoadElem(t, ctx, mc, collName)
return collName, opt
}
func semanticRows(opt hp.StructAElementSchemaOption) []hp.StructARow {
return []hp.StructARow{
{
ID: 0, DocInt: 0,
NormalVector: hp.SeedVector(99990, opt.Dim),
StructA: []hp.StructAElement{
{Embedding: hp.SeedVector(0, opt.Dim), IntVal: 1, StrVal: "a", Color: "Red", Size: "S"},
{Embedding: hp.SeedVector(1, opt.Dim), IntVal: 2, StrVal: "b", Color: "Blue", Size: "L"},
{Embedding: hp.SeedVector(2, opt.Dim), IntVal: 3, StrVal: "c", Color: "Green", Size: "M"},
},
},
{
ID: 1, DocInt: 1,
NormalVector: hp.SeedVector(99991, opt.Dim),
StructA: []hp.StructAElement{
{Embedding: hp.SeedVector(10, opt.Dim), IntVal: 1, StrVal: "x", Color: "Red", Size: "L"},
{Embedding: hp.SeedVector(11, opt.Dim), IntVal: 2, StrVal: "y", Color: "Red", Size: "L"},
},
},
{
ID: 2, DocInt: 2,
NormalVector: hp.SeedVector(99992, opt.Dim),
StructA: []hp.StructAElement{
{Embedding: hp.SeedVector(20, opt.Dim), IntVal: 1, StrVal: "p", Color: "Blue", Size: "S"},
{Embedding: hp.SeedVector(21, opt.Dim), IntVal: 2, StrVal: "q", Color: "Green", Size: "XL"},
},
},
}
}
func TestStructArrayElementMatchSearch(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
opt := hp.DefaultStructAElementSchemaOption("")
opt.IncludeDocVChar = false
opt.IncludeCategory = false
opt.IncludeSize = true
opt.IncludeFloatVal = true
t.Run("match_all_basic", func(t *testing.T) {
ds := hp.GenerateStructAElementData(500, 0, opt)
collName := common.GenRandomString(elemSearchPrefix+"_ma_basic", 6)
o2 := opt
o2.CollectionName = collName
schema, structSchema := hp.CreateStructAElementSchema(o2)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
insertElemDataset(t, ctx, mc, collName, structSchema, ds, o2)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
indexAndLoadElem(t, ctx, mc, collName)
ids := queryAllIDs(t, ctx, mc, collName, `MATCH_ALL(structA, $[color] == "Red")`, 100)
gt := hp.GtMatch(ds.Rows, "MATCH_ALL", func(e hp.StructAElement) bool { return e.Color == "Red" }, 0, nil)
require.True(t, subset(ids, hp.IDSetToSorted(gt)),
"got %v not subset of gt %v", ids, hp.IDSetToSorted(gt))
})
t.Run("match_all_compound_same_element", func(t *testing.T) {
collName, _ := matchSearchSetup(t, ctx, mc, elemSearchPrefix+"_ma_compound", semanticRows(opt))
ids := queryAllIDs(t, ctx, mc, collName, `MATCH_ALL(structA, $[color] == "Red" && $[size] == "L")`, 100)
require.Contains(t, ids, int64(1), "row 1: all elements are Red+L")
require.NotContains(t, ids, int64(0), "row 0: not all elements are Red+L")
})
t.Run("match_any_basic", func(t *testing.T) {
ds := hp.GenerateStructAElementData(500, 0, opt)
collName := common.GenRandomString(elemSearchPrefix+"_many_basic", 6)
o2 := opt
o2.CollectionName = collName
schema, structSchema := hp.CreateStructAElementSchema(o2)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
insertElemDataset(t, ctx, mc, collName, structSchema, ds, o2)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
indexAndLoadElem(t, ctx, mc, collName)
ids := queryAllIDs(t, ctx, mc, collName, `MATCH_ANY(structA, $[color] == "Blue")`, 100)
gt := hp.GtMatch(ds.Rows, "MATCH_ANY", func(e hp.StructAElement) bool { return e.Color == "Blue" }, 0, nil)
require.True(t, subset(ids, hp.IDSetToSorted(gt)),
"got %v not subset of gt %v", ids, hp.IDSetToSorted(gt))
})
t.Run("match_nested_semantic_verification", func(t *testing.T) {
collName, _ := matchSearchSetup(t, ctx, mc, elemSearchPrefix+"_semantic", semanticRows(opt))
ids := queryAllIDs(t, ctx, mc, collName, `MATCH_ANY(structA, $[color] == "Red" && $[size] == "L")`, 100)
require.NotContains(t, ids, int64(0), "row 0: Red and L on different elements should NOT match")
require.Contains(t, ids, int64(1), "row 1: elem[0]={Red,L} should match")
})
}
// =============================================================================
// 3. TestMilvusClientStructArrayElementNestedIndex (3 L0)
// =============================================================================
// nestedIndexSetup mirrors python `_setup_base_collection` with one extra index on the requested
// struct sub-field, then queries one row to confirm the load+index path works.
func nestedIndexSetup(t *testing.T, ctx CtxT, mc MC, namePrefix, subField, indexType string) {
collName := common.GenRandomString(namePrefix, 6)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeDocVChar = false
opt.IncludeCategory = false
opt.IncludeSize = true
opt.IncludeFloatVal = true
schema, structSchema := hp.CreateStructAElementSchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
ds := hp.GenerateStructAElementData(200, 0, opt)
insertElemDataset(t, ctx, mc, collName, structSchema, ds, opt)
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA["+subField+"]",
index.NewGenericIndex("nested_idx", map[string]string{"index_type": indexType})))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
// Sanity query
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("id < 5").WithOutputFields("id").WithLimit(5).
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 5, rs.ResultCount)
}
func TestStructArrayElementNestedIndexInvertedInt(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
nestedIndexSetup(t, ctx, mc, elemSearchPrefix+"_ni_inv_int", "int_val", "INVERTED")
}
func TestStructArrayElementNestedIndexInvertedVarchar(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
nestedIndexSetup(t, ctx, mc, elemSearchPrefix+"_ni_inv_str", "str_val", "INVERTED")
}
func TestStructArrayElementNestedIndexSTLSortInt(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
nestedIndexSetup(t, ctx, mc, elemSearchPrefix+"_ni_stl_int", "int_val", "STL_SORT")
}
// =============================================================================
// 4. TestMilvusClientStructArrayElementNonFloatVectors (2 L0 — schema create only)
// =============================================================================
func runNonFloatVectorCreate(t *testing.T, ctx CtxT, mc MC, namePrefix string, vecType entity.FieldType, metric entity.MetricType) {
collName := common.GenRandomString(namePrefix, 6)
dim := hp.StructAElemDim
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("embedding").WithDataType(vecType).WithDim(int64(dim))).
WithField(entity.NewField().WithName("int_val").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("str_val").WithDataType(entity.FieldTypeVarChar).WithMaxLength(256))
schema := entity.NewSchema().WithName(collName).
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("doc_int").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("normal_vector").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("structA").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(int64(hp.StructAElemCapacity)).
WithStructSchema(structSchema))
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
_, err := mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(metric, 16, 200)))
common.CheckErr(t, err, true)
}
func TestStructArrayElementNonFloatVectorsFloat16(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
runNonFloatVectorCreate(t, ctx, mc, elemSearchPrefix+"_nf_f16",
entity.FieldTypeFloat16Vector, entity.MaxSimL2)
}
func TestStructArrayElementNonFloatVectorsBFloat16(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
runNonFloatVectorCreate(t, ctx, mc, elemSearchPrefix+"_nf_bf16",
entity.FieldTypeBFloat16Vector, entity.MaxSimIP)
}
// =============================================================================
// 5. TestMilvusClientStructArrayElementGroupBySearch (2 L0)
// =============================================================================
// groupByCollection inlines the GroupBy schema (id + doc_int + doc_category(VarChar) + doc_group(Int32)
// + normal_vector + structA{embedding,int_val,str_val,float_val,color}) since it's unique to this
// suite.
func groupByCollection(t *testing.T, ctx CtxT, mc MC) (string, []hp.StructARow) {
collName := common.GenRandomString(elemSearchPrefix+"_gb", 6)
dim := hp.StructAElemDim
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("embedding").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("int_val").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("str_val").WithDataType(entity.FieldTypeVarChar).WithMaxLength(65535)).
WithField(entity.NewField().WithName("float_val").WithDataType(entity.FieldTypeFloat)).
WithField(entity.NewField().WithName("color").WithDataType(entity.FieldTypeVarChar).WithMaxLength(128))
schema := entity.NewSchema().WithName(collName).
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("doc_int").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("doc_category").WithDataType(entity.FieldTypeVarChar).WithMaxLength(128)).
WithField(entity.NewField().WithName("doc_group").WithDataType(entity.FieldTypeInt32)).
WithField(entity.NewField().WithName("normal_vector").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("structA").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(int64(hp.StructAElemCapacity)).
WithStructSchema(structSchema))
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
const nb = 500
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeDocVChar = false
opt.IncludeCategory = false
opt.IncludeSize = false
opt.IncludeFloatVal = true
ds := hp.GenerateStructAElementData(nb, 0, opt)
rows := ds.Rows
ids := make([]int64, nb)
docInts := make([]int64, nb)
docCats := make([]string, nb)
docGroups := make([]int32, nb)
vectors := make([][]float32, nb)
structRows := make([]map[string]any, nb)
for i, r := range rows {
ids[i] = r.ID
docInts[i] = r.DocInt
docCats[i] = hp.StructAElemCategories[r.ID%4]
docGroups[i] = int32(r.ID % 5)
vectors[i] = r.NormalVector
// build sub-field rows directly (matches GroupBy schema sub-fields)
embs := make([][]float32, len(r.StructA))
intVals := make([]int64, len(r.StructA))
strVals := make([]string, len(r.StructA))
floatVals := make([]float32, len(r.StructA))
colors := make([]string, len(r.StructA))
for j, e := range r.StructA {
embs[j] = e.Embedding
intVals[j] = e.IntVal
strVals[j] = e.StrVal
floatVals[j] = e.FloatVal
colors[j] = e.Color
}
structRows[i] = map[string]any{
"embedding": embs,
"int_val": intVals,
"str_val": strVals,
"float_val": floatVals,
"color": colors,
}
}
docCatCol := column.NewColumnVarChar("doc_category", docCats)
docGroupCol := column.NewColumnInt32("doc_group", docGroups)
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", ids).
WithInt64Column("doc_int", docInts).
WithColumns(docCatCol, docGroupCol).
WithFloatVectorColumn("normal_vector", dim, vectors).
WithStructArrayColumn("structA", structSchema, structRows))
common.CheckErr(t, err, true)
_, err = mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
return collName, rows
}
func TestStructArrayElementGroupByElementFilterBasic(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, rows := groupByCollection(t, ctx, mc)
queryVec := rows[0].NormalVector
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10,
[]entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("normal_vector").
WithFilter(`MATCH_ANY(structA, $[int_val] > 100)`).
WithGroupByField("doc_category").
WithOutputFields("id", "doc_category").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.GreaterOrEqual(t, len(rs), 1)
require.Greater(t, rs[0].ResultCount, 0)
// no duplicate doc_category in returned rows
seen := map[string]bool{}
catCol := rs[0].GetColumn("doc_category")
for i := 0; i < rs[0].ResultCount; i++ {
v, _ := catCol.Get(i)
c := v.(string)
require.False(t, seen[c], "duplicate doc_category %q in grouped results", c)
seen[c] = true
}
}
func TestStructArrayElementGroupByMatchAll(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, rows := groupByCollection(t, ctx, mc)
queryVec := rows[0].NormalVector
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10,
[]entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("normal_vector").
WithFilter(`MATCH_ALL(structA, $[int_val] > 0)`).
WithGroupByField("doc_category").
WithOutputFields("id", "doc_category").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.GreaterOrEqual(t, len(rs), 1)
require.Greater(t, rs[0].ResultCount, 0)
}
// =============================================================================
// 6. TestMilvusClientStructArrayElementSearchNoFilter (4 L0)
// =============================================================================
// noFilterCollection inlines the NoFilter schema (id + doc_int + doc_category + normal_vector +
// structA{embedding,int_val,str_val,color}).
func noFilterCollection(t *testing.T, ctx CtxT, mc MC) (string, []hp.StructARow) {
collName := common.GenRandomString(elemSearchPrefix+"_nf", 6)
dim := hp.StructAElemDim
const nb = 500
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("embedding").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("int_val").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("str_val").WithDataType(entity.FieldTypeVarChar).WithMaxLength(65535)).
WithField(entity.NewField().WithName("color").WithDataType(entity.FieldTypeVarChar).WithMaxLength(128))
schema := entity.NewSchema().WithName(collName).
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("doc_int").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("doc_category").WithDataType(entity.FieldTypeVarChar).WithMaxLength(128)).
WithField(entity.NewField().WithName("normal_vector").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("structA").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(int64(hp.StructAElemCapacity)).
WithStructSchema(structSchema))
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema).WithConsistencyLevel(entity.ClStrong)), true)
opt := hp.DefaultStructAElementSchemaOption(collName)
opt.IncludeDocVChar = false
opt.IncludeCategory = false
opt.IncludeSize = false
opt.IncludeFloatVal = false
ds := hp.GenerateStructAElementData(nb, 0, opt)
rows := ds.Rows
ids := make([]int64, nb)
docInts := make([]int64, nb)
docCats := make([]string, nb)
vectors := make([][]float32, nb)
structRows := make([]map[string]any, nb)
for i, r := range rows {
ids[i] = r.ID
docInts[i] = r.DocInt
docCats[i] = hp.StructAElemCategories[r.ID%4]
vectors[i] = r.NormalVector
embs := make([][]float32, len(r.StructA))
intVals := make([]int64, len(r.StructA))
strVals := make([]string, len(r.StructA))
colors := make([]string, len(r.StructA))
for j, e := range r.StructA {
embs[j] = e.Embedding
intVals[j] = e.IntVal
strVals[j] = e.StrVal
colors[j] = e.Color
}
structRows[i] = map[string]any{
"embedding": embs,
"int_val": intVals,
"str_val": strVals,
"color": colors,
}
}
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", ids).
WithInt64Column("doc_int", docInts).
WithVarcharColumn("doc_category", docCats).
WithFloatVectorColumn("normal_vector", dim, vectors).
WithStructArrayColumn("structA", structSchema, structRows))
common.CheckErr(t, err, true)
_, err = mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
// Plain COSINE on sub-vector so single-vector searches work directly.
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "structA[embedding]",
index.NewHNSWIndex(entity.COSINE, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
return collName, rows
}
func TestStructArrayElementSearchNoFilter(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, rows := noFilterCollection(t, ctx, mc)
t.Run("basic", func(t *testing.T) {
queryVec := rows[0].StructA[0].Embedding
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10, []entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("structA[embedding]").
WithSearchParam("metric_type", "COSINE").
WithOutputFields("id").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 10, rs[0].ResultCount, "expected exactly limit=10 rows")
first, _ := rs[0].GetColumn("id").Get(0)
require.EqualValues(t, int64(0), first.(int64), "self-match top-1 should be row 0")
})
t.Run("ground_truth", func(t *testing.T) {
queryVec := rows[42].StructA[1].Embedding
const limit = 20
rs, err := mc.Search(ctx, client.NewSearchOption(collName, limit, []entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("structA[embedding]").
WithSearchParam("metric_type", "COSINE").
WithOutputFields("id").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
// HNSW recall on a small dataset can return slightly fewer than limit; accept ≥ 90%.
require.GreaterOrEqual(t, rs[0].ResultCount, limit*9/10,
"got %d results, expected at least %d", rs[0].ResultCount, limit*9/10)
gtIDs := hp.GtElementSearchNoFilter(rows, queryVec, "COSINE", limit)
idCol := rs[0].GetColumn("id")
got := make([]int64, rs[0].ResultCount)
for i := 0; i < rs[0].ResultCount; i++ {
v, _ := idCol.Get(i)
got[i] = v.(int64)
}
require.EqualValues(t, gtIDs[0], got[0], "top-1 must match ground truth")
// top-K recall ≥ 0.85 (HNSW recall + small-dataset tolerance)
gtSet := map[int64]bool{}
for _, id := range gtIDs {
gtSet[id] = true
}
overlap := 0
for _, id := range got {
if gtSet[id] {
overlap++
}
}
require.GreaterOrEqual(t, float64(overlap)/float64(limit), 0.85,
"recall too low: %d/%d", overlap, limit)
})
t.Run("distance_order", func(t *testing.T) {
queryVec := rows[0].StructA[0].Embedding
const limit = 50
rs, err := mc.Search(ctx, client.NewSearchOption(collName, limit, []entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("structA[embedding]").
WithSearchParam("metric_type", "COSINE").
WithOutputFields("id").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.GreaterOrEqual(t, rs[0].ResultCount, limit*9/10,
"got %d results, expected at least %d", rs[0].ResultCount, limit*9/10)
// COSINE: distances should be monotonically non-increasing
for i := 0; i < rs[0].ResultCount-1; i++ {
require.GreaterOrEqual(t, float64(rs[0].Scores[i]+1e-3), float64(rs[0].Scores[i+1]),
"distance not monotonic at pos %d: %f < %f", i, rs[0].Scores[i], rs[0].Scores[i+1])
}
})
t.Run("group_by_pk", func(t *testing.T) {
queryVec := rows[0].StructA[0].Embedding
const limit = 20
rs, err := mc.Search(ctx, client.NewSearchOption(collName, limit, []entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("structA[embedding]").
WithSearchParam("metric_type", "COSINE").
WithGroupByField("id").
WithOutputFields("id").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, limit, rs[0].ResultCount)
seen := map[int64]bool{}
idCol := rs[0].GetColumn("id")
for i := 0; i < rs[0].ResultCount; i++ {
v, _ := idCol.Get(i)
id := v.(int64)
require.False(t, seen[id], "duplicate PK %d under group_by=id", id)
seen[id] = true
}
first, _ := idCol.Get(0)
require.EqualValues(t, int64(0), first.(int64), "self-match top-1 should be row 0 even with group_by_pk")
})
}
tests/go_client/testcases/struct_array_test.go
+555
View File
@@ -0,0 +1,555 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Tests in this file mirror the L0 (smoke / must-pass) cases from
// tests/python_client/milvus_client/test_milvus_client_struct_array.py.
// Each Go test function is named after the original Python test it ports.
package testcases
import (
"context"
"fmt"
"math/rand"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus/client/v2/column"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/client/v2/index"
client "github.com/milvus-io/milvus/client/v2/milvusclient"
"github.com/milvus-io/milvus/tests/go_client/base"
"github.com/milvus-io/milvus/tests/go_client/common"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
)
const structArrayTestNb = 200 // shrunk from python's default_nb=3000 for faster Go SDK runs
// canonicalStructArrayCollection creates the canonical schema (id + normal_vector + clips with
// clip_str/clip_embedding1/clip_embedding2), inserts numRows of random data, builds indexes on
// normal_vector and the two struct sub-vectors, and loads the collection.
//
// Returns the collection name, struct schema (needed for WithStructArrayColumn), and the
// generated test data so callers can run further assertions.
func canonicalStructArrayCollection(t *testing.T, ctx CtxT, mc MC, numRows int) (string, *entity.StructSchema, hp.StructArrayTestData) {
collName := common.GenRandomString(hp.StructArrayPrefix, 6)
opt := hp.DefaultStructArraySchemaOption(collName)
schema, structSchema := hp.CreateStructArraySchema(opt)
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema).
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
data := hp.GenerateStructArrayData(numRows, opt)
insertOpt := client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", data.IDs).
WithFloatVectorColumn("normal_vector", data.Dim, data.NormalVectors).
WithStructArrayColumn("clips", structSchema, data.ClipsRows)
_, err = mc.Insert(ctx, insertOpt)
common.CheckErr(t, err, true)
_, err = mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
indexAndLoad(t, ctx, mc, collName, data.Dim)
return collName, structSchema, data
}
// indexAndLoad builds the canonical 3 indexes (normal_vector + 2 sub-vectors) and loads.
func indexAndLoad(t *testing.T, ctx CtxT, mc MC, collName string, dim int) {
_, err := mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewIvfFlatIndex(entity.L2, 128)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "clips[clip_embedding1]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "clips[clip_embedding2]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
}
// type aliases to keep test signatures readable
type (
CtxT = context.Context
MC = *base.MilvusClient
)
// TestStructArrayCreateWithClipEmbedding1 ports test_create_struct_array_with_clip_embedding1.
func TestStructArrayCreateWithClipEmbedding1(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(hp.StructArrayPrefix+"_basic", 6)
schema, _ := hp.CreateStructArraySchema(hp.DefaultStructArraySchemaOption(collName))
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema)), true)
has, err := mc.HasCollection(ctx, client.NewHasCollectionOption(collName))
common.CheckErr(t, err, true)
require.True(t, has)
}
// TestStructArrayCreateWithScalarFields ports test_create_struct_array_with_scalar_fields.
func TestStructArrayCreateWithScalarFields(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(hp.StructArrayPrefix+"_basic", 6)
dim := hp.StructArrayDefaultDim
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("int_field").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("float_field").WithDataType(entity.FieldTypeFloat)).
WithField(entity.NewField().WithName("string_field").WithDataType(entity.FieldTypeVarChar).WithMaxLength(512)).
WithField(entity.NewField().WithName("bool_field").WithDataType(entity.FieldTypeBool))
schema := entity.NewSchema().WithName(collName).
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("normal_vector").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("metadata").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(int64(hp.StructArrayDefaultCapacity)).
WithStructSchema(structSchema))
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema)), true)
has, err := mc.HasCollection(ctx, client.NewHasCollectionOption(collName))
common.CheckErr(t, err, true)
require.True(t, has)
}
// TestStructArrayInsertBasic ports test_insert_struct_array_basic.
func TestStructArrayInsertBasic(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(hp.StructArrayPrefix+"_basic", 6)
opt := hp.DefaultStructArraySchemaOption(collName)
schema, structSchema := hp.CreateStructArraySchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema)), true)
data := hp.GenerateStructArrayData(structArrayTestNb, opt)
res, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", data.IDs).
WithFloatVectorColumn("normal_vector", data.Dim, data.NormalVectors).
WithStructArrayColumn("clips", structSchema, data.ClipsRows))
common.CheckErr(t, err, true)
require.EqualValues(t, structArrayTestNb, res.InsertCount)
}
// TestStructArrayCreateEmbListHNSWIndexCosine ports test_create_emb_list_hnsw_index_cosine.
func TestStructArrayCreateEmbListHNSWIndexCosine(t *testing.T) {
runEmbListHNSWIndex(t, entity.MaxSimCosine)
}
// TestStructArrayCreateEmbListHNSWIndexIp ports test_create_emb_list_hnsw_index_ip.
// Python test name says _ip but the body actually uses MAX_SIM_COSINE; we mirror that.
func TestStructArrayCreateEmbListHNSWIndexIp(t *testing.T) {
runEmbListHNSWIndex(t, entity.MaxSimCosine)
}
func runEmbListHNSWIndex(t *testing.T, metric entity.MetricType) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(hp.StructArrayPrefix+"_index", 6)
opt := hp.DefaultStructArraySchemaOption(collName)
schema, structSchema := hp.CreateStructArraySchema(opt)
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema)), true)
data := hp.GenerateStructArrayData(structArrayTestNb, opt)
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", data.IDs).
WithFloatVectorColumn("normal_vector", data.Dim, data.NormalVectors).
WithStructArrayColumn("clips", structSchema, data.ClipsRows))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewIvfFlatIndex(entity.L2, 128)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "clips[clip_embedding1]",
index.NewHNSWIndex(metric, 16, 200)))
common.CheckErr(t, err, true)
}
// TestStructArraySearchVectorSingle ports test_search_struct_array_vector_single.
// Original uses EmbeddingList with one vector — we use entity.FloatVectorArray with one element.
func TestStructArraySearchVectorSingle(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, _, data := canonicalStructArrayCollection(t, ctx, mc, structArrayTestNb)
// baseline: search normal vector field
queryVec := hp.RandFloatVector(data.Dim)
normalRS, err := mc.Search(ctx, client.NewSearchOption(collName, 10,
[]entity.Vector{entity.FloatVector(queryVec)}).
WithANNSField("normal_vector").
WithSearchParam("nprobe", "10").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.Greater(t, normalRS[0].ResultCount, 0)
// MAX_SIM search on struct sub-vector with EmbList(=FloatVectorArray) of one vector
embList := entity.FloatVectorArray{entity.FloatVector(queryVec)}
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10, []entity.Vector{embList}).
WithANNSField("clips[clip_embedding1]").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.Greater(t, rs[0].ResultCount, 0)
}
// TestStructArraySearchVectorMultiple ports test_search_struct_array_vector_multiple.
func TestStructArraySearchVectorMultiple(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, _, data := canonicalStructArrayCollection(t, ctx, mc, structArrayTestNb)
embList := entity.FloatVectorArray{
entity.FloatVector(hp.RandFloatVector(data.Dim)),
entity.FloatVector(hp.RandFloatVector(data.Dim)),
entity.FloatVector(hp.RandFloatVector(data.Dim)),
}
rs, err := mc.Search(ctx, client.NewSearchOption(collName, 10, []entity.Vector{embList}).
WithANNSField("clips[clip_embedding1]").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.Greater(t, rs[0].ResultCount, 0)
}
// TestStructArrayHybridSearchWithNormalVector ports
// test_hybrid_search_struct_array_with_normal_vector.
func TestStructArrayHybridSearchWithNormalVector(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(hp.StructArrayPrefix+"_hybrid", 6)
dim := hp.StructArrayDefaultDim
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("clip_str").WithDataType(entity.FieldTypeVarChar).WithMaxLength(65535)).
WithField(entity.NewField().WithName("clip_embedding").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim)))
schema := entity.NewSchema().WithName(collName).
WithField(entity.NewField().WithName("pk").WithDataType(entity.FieldTypeVarChar).WithMaxLength(100).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("random").WithDataType(entity.FieldTypeDouble)).
WithField(entity.NewField().WithName("embeddings").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("clips").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(100).
WithStructSchema(structSchema))
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema)), true)
const numEntities = 100
pks := make([]string, numEntities)
randoms := make([]float64, numEntities)
embeddings := make([][]float32, numEntities)
rows := make([]map[string]any, numEntities)
for i := 0; i < numEntities; i++ {
pks[i] = fmt.Sprintf("%d", i)
randoms[i] = rand.Float64()
embeddings[i] = hp.RandFloatVector(dim)
count := 2 + rand.Intn(2) // 2 or 3
strs := make([]string, count)
embs := make([][]float32, count)
for j := 0; j < count; j++ {
strs[j] = fmt.Sprintf("item_%d_%d", i, j)
embs[j] = hp.RandFloatVector(dim)
}
rows[i] = map[string]any{"clip_str": strs, "clip_embedding": embs}
}
pkCol := column.NewColumnVarChar("pk", pks)
randomCol := column.NewColumnDouble("random", randoms)
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(collName).
WithColumns(pkCol, randomCol).
WithFloatVectorColumn("embeddings", dim, embeddings).
WithStructArrayColumn("clips", structSchema, rows))
common.CheckErr(t, err, true)
_, err = mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "embeddings",
index.NewIvfFlatIndex(entity.L2, 128)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "clips[clip_embedding]",
index.NewHNSWIndex(entity.MaxSimL2, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
queryVec := entity.FloatVector(hp.RandFloatVector(dim))
queryEmbList := entity.FloatVectorArray{entity.FloatVector(hp.RandFloatVector(dim))}
rs, err := mc.HybridSearch(ctx, client.NewHybridSearchOption(collName, 5,
client.NewAnnRequest("embeddings", 5, queryVec),
client.NewAnnRequest("clips[clip_embedding]", 5, queryEmbList),
).WithReranker(client.NewRRFReranker()).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.GreaterOrEqual(t, len(rs), 1)
require.Greater(t, rs[0].ResultCount, 0)
}
// TestStructArrayQueryAllFields ports test_query_struct_array_all_fields.
func TestStructArrayQueryAllFields(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, _, _ := canonicalStructArrayCollection(t, ctx, mc, structArrayTestNb)
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("id >= 0").WithLimit(10).
WithOutputFields("*").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.Greater(t, rs.ResultCount, 0)
clipsCol := rs.GetColumn("clips")
require.NotNil(t, clipsCol, "clips column must be present in query results")
for i := 0; i < rs.ResultCount; i++ {
v, err := clipsCol.Get(i)
require.NoError(t, err)
_, ok := v.(map[string]any)
require.True(t, ok, "struct array element must decode as map[string]any")
}
}
// TestStructArrayQuerySpecificFields ports test_query_struct_array_specific_fields.
func TestStructArrayQuerySpecificFields(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, _, _ := canonicalStructArrayCollection(t, ctx, mc, structArrayTestNb)
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("id >= 0").WithLimit(10).
WithOutputFields("id", "clips").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.Greater(t, rs.ResultCount, 0)
require.NotNil(t, rs.GetColumn("id"))
require.NotNil(t, rs.GetColumn("clips"))
}
// TestStructArrayUpsertData ports test_upsert_struct_array_data.
// Scaled-down: 200 flushed + 100 growing + 5 upsert per segment, vs python's 2000 + 1000 + 10.
func TestStructArrayUpsertData(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, structSchema, _ := crudCollection(t, ctx, mc)
dim := hp.StructArrayDefaultDim
insertSegment := func(start, count int, label string) {
ids := make([]int64, count)
vecs := make([][]float32, count)
rows := make([]map[string]any, count)
for i := 0; i < count; i++ {
ids[i] = int64(start + i)
vecs[i] = hp.RandFloatVector(dim)
rows[i] = map[string]any{
"clip_embedding1": [][]float32{hp.RandFloatVector(dim)},
"scalar_field": []int64{int64(start + i)},
"label": []string{fmt.Sprintf("%s_%d", label, start+i)},
}
}
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", ids).
WithFloatVectorColumn("normal_vector", dim, vecs).
WithStructArrayColumn("clips", structSchema, rows))
common.CheckErr(t, err, true)
}
insertSegment(0, 200, "flushed")
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
insertSegment(200, 100, "growing")
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewIvfFlatIndex(entity.L2, 128)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "clips[clip_embedding1]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
// Upsert 5 from flushed (ids 0..4) and 5 from growing (200..204).
upsertIDs := []int64{0, 1, 2, 3, 4, 200, 201, 202, 203, 204}
upsertVecs := make([][]float32, len(upsertIDs))
upsertRows := make([]map[string]any, len(upsertIDs))
for i, id := range upsertIDs {
upsertVecs[i] = hp.RandFloatVector(dim)
var prefix string
if id < 200 {
prefix = "updated_flushed"
} else {
prefix = "updated_growing"
}
upsertRows[i] = map[string]any{
"clip_embedding1": [][]float32{hp.RandFloatVector(dim)},
"scalar_field": []int64{id + 10000},
"label": []string{fmt.Sprintf("%s_%d", prefix, id)},
}
}
_, err = mc.Upsert(ctx, client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", upsertIDs).
WithFloatVectorColumn("normal_vector", dim, upsertVecs).
WithStructArrayColumn("clips", structSchema, upsertRows))
common.CheckErr(t, err, true)
// Skip second flush: target instance has flush rate limited at 0.1/s. Rely on Strong
// consistency in the query to see upsert results regardless of segment state.
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("id < 5").WithOutputFields("id", "clips").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 5, rs.ResultCount)
clips := rs.GetColumn("clips")
for i := 0; i < rs.ResultCount; i++ {
v, err := clips.Get(i)
require.NoError(t, err)
m := v.(map[string]any)
labels := m["label"].([]string)
require.Len(t, labels, 1)
require.True(t, strings.Contains(labels[0], "updated_flushed"),
"row %d label=%s does not contain updated_flushed", i, labels[0])
}
rs2, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("id >= 200 and id < 205").WithOutputFields("id", "clips").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 5, rs2.ResultCount)
}
// TestStructArrayDeleteData ports test_delete_struct_array_data.
func TestStructArrayDeleteData(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, structSchema, _ := crudCollection(t, ctx, mc)
dim := hp.StructArrayDefaultDim
insertSegment := func(start, count int, label string) {
ids := make([]int64, count)
vecs := make([][]float32, count)
rows := make([]map[string]any, count)
for i := 0; i < count; i++ {
ids[i] = int64(start + i)
vecs[i] = hp.RandFloatVector(dim)
rows[i] = map[string]any{
"clip_embedding1": [][]float32{hp.RandFloatVector(dim)},
"scalar_field": []int64{int64(start + i)},
"label": []string{fmt.Sprintf("%s_%d", label, start+i)},
}
}
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(collName).
WithInt64Column("id", ids).
WithFloatVectorColumn("normal_vector", dim, vecs).
WithStructArrayColumn("clips", structSchema, rows))
common.CheckErr(t, err, true)
}
insertSegment(0, 200, "flushed")
_, err := mc.Flush(ctx, client.NewFlushOption(collName))
common.CheckErr(t, err, true)
insertSegment(200, 100, "growing")
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "normal_vector",
index.NewIvfFlatIndex(entity.L2, 128)))
common.CheckErr(t, err, true)
_, err = mc.CreateIndex(ctx, client.NewCreateIndexOption(collName, "clips[clip_embedding1]",
index.NewHNSWIndex(entity.MaxSimCosine, 16, 200)))
common.CheckErr(t, err, true)
loadTask, err := mc.LoadCollection(ctx, client.NewLoadCollectionOption(collName))
common.CheckErr(t, err, true)
common.CheckErr(t, loadTask.Await(ctx), true)
// delete 5 from flushed (ids 0..4) and 5 from growing (200..204)
_, err = mc.Delete(ctx, client.NewDeleteOption(collName).WithExpr("id in [0,1,2,3,4,200,201,202,203,204]"))
common.CheckErr(t, err, true)
// Skip second flush: target instance has flush rate limited at 0.1/s. Rely on Strong
// consistency in the subsequent query.
rs, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("id in [0,1,2,3,4,200,201,202,203,204]").
WithOutputFields("id").
WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.EqualValues(t, 0, rs.ResultCount, "deleted rows should not be returned")
}
// crudCollection creates the CRUD-suite collection (clips with clip_embedding1 + scalar_field +
// label, normal_vector nullable).
func crudCollection(t *testing.T, ctx CtxT, mc MC) (string, *entity.StructSchema, *entity.Schema) {
collName := common.GenRandomString(hp.StructArrayPrefix+"_crud", 6)
dim := hp.StructArrayDefaultDim
structSchema := entity.NewStructSchema().
WithField(entity.NewField().WithName("clip_embedding1").WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("scalar_field").WithDataType(entity.FieldTypeInt64)).
WithField(entity.NewField().WithName("label").WithDataType(entity.FieldTypeVarChar).WithMaxLength(128))
schema := entity.NewSchema().WithName(collName).
WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).
WithField(entity.NewField().WithName("normal_vector").
WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))).
WithField(entity.NewField().WithName("clips").
WithDataType(entity.FieldTypeArray).
WithElementType(entity.FieldTypeStruct).
WithMaxCapacity(100).
WithStructSchema(structSchema))
common.CheckErr(t, mc.CreateCollection(ctx,
client.NewCreateCollectionOption(collName, schema)), true)
return collName, structSchema, schema
}
// TestStructArrayRangeSearchNotSupported ports test_struct_array_range_search_not_supported.
func TestStructArrayRangeSearchNotSupported(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := hp.CreateDefaultMilvusClient(ctx, t)
collName, _, data := canonicalStructArrayCollection(t, ctx, mc, structArrayTestNb)
queryEmb := entity.FloatVectorArray{entity.FloatVector(hp.RandFloatVector(data.Dim))}
// Range params must be embedded in the "params" JSON; the server rejects range search on
// struct sub-vector regardless of metric.
_, err := mc.Search(ctx, client.NewSearchOption(collName, 10, []entity.Vector{queryEmb}).
WithANNSField("clips[clip_embedding1]").
WithSearchParam("params", `{"radius": 0.1, "range_filter": 0.5}`).
WithConsistencyLevel(entity.ClStrong))
require.Error(t, err, "range search on struct sub-vector should be rejected")
require.Contains(t, err.Error(), "range search")
}