mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
fix: Validate REST quick-create enum fields
Signed-off-by: Yanliang Qiao <yanliang.qiao@zilliz.com>
This commit is contained in:
@@ -44,6 +44,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/proxy"
|
||||
"github.com/milvus-io/milvus/internal/types"
|
||||
"github.com/milvus-io/milvus/internal/util/hookutil"
|
||||
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/metrics"
|
||||
"github.com/milvus-io/milvus/pkg/v3/mlog"
|
||||
@@ -54,6 +55,7 @@ import (
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/externalspec"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/metric"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/requestutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
@@ -2046,6 +2048,70 @@ func (h *HandlersV2) advancedSearch(ctx context.Context, c *gin.Context, anyReq
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func defaultMetricTypeForQuickCreate(dataType schemapb.DataType) string {
|
||||
switch dataType {
|
||||
case schemapb.DataType_BinaryVector:
|
||||
return paramtable.BinaryVectorDefaultMetricType
|
||||
case schemapb.DataType_SparseFloatVector:
|
||||
return paramtable.SparseFloatVectorDefaultMetricType
|
||||
default:
|
||||
return DefaultMetricType
|
||||
}
|
||||
}
|
||||
|
||||
func validateMetricTypeForQuickCreate(dataType schemapb.DataType, metricType string) error {
|
||||
switch dataType {
|
||||
case schemapb.DataType_BinaryVector:
|
||||
if !funcutil.SliceContain(indexparamcheck.BinaryVectorMetrics, metricType) {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "binary vector index does not support metric type: "+metricType)
|
||||
}
|
||||
case schemapb.DataType_SparseFloatVector:
|
||||
if !funcutil.SliceContain(indexparamcheck.SparseFloatVectorMetrics, metricType) {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "only IP&BM25 is the supported metric type for sparse index")
|
||||
}
|
||||
if metricType == metric.BM25 {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "only BM25 Function output field support BM25 metric type")
|
||||
}
|
||||
default:
|
||||
if !funcutil.SliceContain(indexparamcheck.FloatVectorMetrics, metricType) {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "float vector index does not support metric type: "+metricType)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func consistencyLevelForCreateCollection(httpReq *CollectionReq) (commonpb.ConsistencyLevel, error) {
|
||||
consistencyLevel := commonpb.ConsistencyLevel_Bounded
|
||||
paramConsistencyLevel, hasParamConsistencyLevel := httpReq.Params["consistencyLevel"]
|
||||
paramConsistencyLevelStr := ""
|
||||
if hasParamConsistencyLevel {
|
||||
paramConsistencyLevelStr = fmt.Sprintf("%s", paramConsistencyLevel)
|
||||
}
|
||||
|
||||
if httpReq.ConsistencyLevel != "" && hasParamConsistencyLevel && httpReq.ConsistencyLevel != paramConsistencyLevelStr {
|
||||
return consistencyLevel, merr.WrapErrParameterInvalid("same consistencyLevel", paramConsistencyLevel,
|
||||
"top-level consistencyLevel conflicts with params.consistencyLevel")
|
||||
}
|
||||
|
||||
consistencyLevelStr := httpReq.ConsistencyLevel
|
||||
var actualConsistencyLevel interface{} = httpReq.ConsistencyLevel
|
||||
if consistencyLevelStr == "" && hasParamConsistencyLevel {
|
||||
consistencyLevelStr = paramConsistencyLevelStr
|
||||
actualConsistencyLevel = paramConsistencyLevel
|
||||
}
|
||||
|
||||
if consistencyLevelStr == "" {
|
||||
return consistencyLevel, nil
|
||||
}
|
||||
|
||||
if level, ok := commonpb.ConsistencyLevel_value[consistencyLevelStr]; ok {
|
||||
return commonpb.ConsistencyLevel(level), nil
|
||||
}
|
||||
|
||||
return consistencyLevel, merr.WrapErrParameterInvalid("Strong, Session, Bounded, Eventually, Customized", actualConsistencyLevel,
|
||||
"consistencyLevel can only be [Strong, Session, Bounded, Eventually, Customized], default: Bounded")
|
||||
}
|
||||
|
||||
func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
|
||||
httpReq := anyReq.(*CollectionReq)
|
||||
req := &milvuspb.CreateCollectionRequest{
|
||||
@@ -2070,6 +2136,7 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
|
||||
var err error
|
||||
fieldNames := map[string]bool{}
|
||||
partitionsNum := int64(-1)
|
||||
quickCreateVectorDataType := schemapb.DataType_FloatVector
|
||||
if len(httpReq.Schema.Fields) == 0 {
|
||||
if httpReq.GetExternalSource() != "" || httpReq.GetExternalSpec() != "" {
|
||||
err := merr.WrapErrParameterInvalid("schema.fields", "empty schema fields",
|
||||
@@ -2092,7 +2159,40 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if httpReq.Dimension == 0 {
|
||||
switch httpReq.VectorFieldType {
|
||||
case "", "FloatVector":
|
||||
httpReq.VectorFieldType = "FloatVector"
|
||||
case "BinaryVector":
|
||||
quickCreateVectorDataType = schemapb.DataType_BinaryVector
|
||||
case "Float16Vector":
|
||||
quickCreateVectorDataType = schemapb.DataType_Float16Vector
|
||||
case "BFloat16Vector":
|
||||
quickCreateVectorDataType = schemapb.DataType_BFloat16Vector
|
||||
case "SparseFloatVector":
|
||||
quickCreateVectorDataType = schemapb.DataType_SparseFloatVector
|
||||
default:
|
||||
err := merr.WrapErrParameterInvalid("FloatVector, BinaryVector, Float16Vector, BFloat16Vector, SparseFloatVector", httpReq.VectorFieldType,
|
||||
"vectorFieldType can only be [FloatVector, BinaryVector, Float16Vector, BFloat16Vector, SparseFloatVector], default: FloatVector")
|
||||
mlog.Warn(ctx, "high level restful api, quickly create collection fail", mlog.Err(err), mlog.Any("request", anyReq))
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{
|
||||
HTTPReturnCode: merr.Code(err),
|
||||
HTTPReturnMessage: err.Error(),
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if quickCreateVectorDataType == schemapb.DataType_SparseFloatVector && httpReq.Dimension != 0 {
|
||||
err := merr.WrapErrParameterInvalid(int32(0), httpReq.Dimension,
|
||||
"dimension should not be specified for SparseFloatVector quick create")
|
||||
mlog.Warn(ctx, "high level restful api, quickly create collection fail", mlog.Err(err), mlog.Any("request", anyReq))
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{
|
||||
HTTPReturnCode: merr.Code(err),
|
||||
HTTPReturnMessage: err.Error(),
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if quickCreateVectorDataType != schemapb.DataType_SparseFloatVector && httpReq.Dimension == 0 {
|
||||
err := merr.WrapErrParameterInvalid("collectionName & dimension", "collectionName",
|
||||
"dimension is required for quickly create collection(default metric type: "+DefaultMetricType+")")
|
||||
mlog.Warn(ctx, "high level restful api, quickly create collection fail", mlog.Err(err), mlog.Any("request", anyReq))
|
||||
@@ -2102,6 +2202,17 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
if len(httpReq.MetricType) == 0 {
|
||||
httpReq.MetricType = defaultMetricTypeForQuickCreate(quickCreateVectorDataType)
|
||||
}
|
||||
if err := validateMetricTypeForQuickCreate(quickCreateVectorDataType, httpReq.MetricType); err != nil {
|
||||
mlog.Warn(ctx, "high level restful api, quickly create collection fail", mlog.Err(err), mlog.Any("request", anyReq))
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{
|
||||
HTTPReturnCode: merr.Code(err),
|
||||
HTTPReturnMessage: err.Error(),
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
idDataType := schemapb.DataType_Int64
|
||||
idParams := []*commonpb.KeyValuePair{}
|
||||
switch httpReq.IDType {
|
||||
@@ -2142,6 +2253,13 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
vectorTypeParams := []*commonpb.KeyValuePair{}
|
||||
if quickCreateVectorDataType != schemapb.DataType_SparseFloatVector {
|
||||
vectorTypeParams = append(vectorTypeParams, &commonpb.KeyValuePair{
|
||||
Key: Dim,
|
||||
Value: strconv.FormatInt(int64(httpReq.Dimension), 10),
|
||||
})
|
||||
}
|
||||
schema, err = proto.Marshal(&schemapb.CollectionSchema{
|
||||
Name: httpReq.CollectionName,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
@@ -2157,14 +2275,9 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
|
||||
FieldID: common.StartOfUserFieldID + 1,
|
||||
Name: httpReq.VectorFieldName,
|
||||
IsPrimaryKey: false,
|
||||
DataType: schemapb.DataType_FloatVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: Dim,
|
||||
Value: strconv.FormatInt(int64(httpReq.Dimension), 10),
|
||||
},
|
||||
},
|
||||
AutoID: DisableAutoID,
|
||||
DataType: quickCreateVectorDataType,
|
||||
TypeParams: vectorTypeParams,
|
||||
AutoID: DisableAutoID,
|
||||
},
|
||||
},
|
||||
EnableDynamicField: enableDynamic,
|
||||
@@ -2314,20 +2427,14 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
|
||||
}
|
||||
req.ShardsNum = shardsNum
|
||||
|
||||
consistencyLevel := commonpb.ConsistencyLevel_Bounded
|
||||
if _, ok := httpReq.Params["consistencyLevel"]; ok {
|
||||
if level, ok := commonpb.ConsistencyLevel_value[fmt.Sprintf("%s", httpReq.Params["consistencyLevel"])]; ok {
|
||||
consistencyLevel = commonpb.ConsistencyLevel(level)
|
||||
} else {
|
||||
err := merr.WrapErrParameterInvalid("Strong, Session, Bounded, Eventually, Customized", httpReq.Params["consistencyLevel"],
|
||||
"consistencyLevel can only be [Strong, Session, Bounded, Eventually, Customized], default: Bounded")
|
||||
mlog.Warn(ctx, "high level restful api, create collection fail", mlog.Err(err), mlog.Any("request", anyReq))
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{
|
||||
HTTPReturnCode: merr.Code(err),
|
||||
HTTPReturnMessage: err.Error(),
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
consistencyLevel, err := consistencyLevelForCreateCollection(httpReq)
|
||||
if err != nil {
|
||||
mlog.Warn(ctx, "high level restful api, create collection fail", mlog.Err(err), mlog.Any("request", anyReq))
|
||||
HTTPAbortReturn(c, http.StatusOK, gin.H{
|
||||
HTTPReturnCode: merr.Code(err),
|
||||
HTTPReturnMessage: err.Error(),
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
req.ConsistencyLevel = consistencyLevel
|
||||
|
||||
@@ -2404,9 +2511,6 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
|
||||
return resp, err
|
||||
}
|
||||
if len(httpReq.Schema.Fields) == 0 {
|
||||
if len(httpReq.MetricType) == 0 {
|
||||
httpReq.MetricType = DefaultMetricType
|
||||
}
|
||||
createIndexReq := &milvuspb.CreateIndexRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: httpReq.CollectionName,
|
||||
|
||||
@@ -2064,6 +2064,272 @@ func TestCreateCollection(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestCreateCollectionQuickVectorFieldType(t *testing.T) {
|
||||
paramtable.Init()
|
||||
paramtable.Get().Save(paramtable.Get().QuotaConfig.QuotaAndLimitsEnabled.Key, "false")
|
||||
defer paramtable.Get().Reset(paramtable.Get().QuotaConfig.QuotaAndLimitsEnabled.Key)
|
||||
|
||||
path := versionalV2(CollectionCategory, CreateAction)
|
||||
|
||||
t.Run("reject invalid vector field type", func(t *testing.T) {
|
||||
proxy := &externalCollectionRESTProxy{}
|
||||
testEngine := initHTTPServerV2(proxy, false)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(`{
|
||||
"collectionName": "book",
|
||||
"dimension": 4,
|
||||
"vectorFieldType": "InvalidVectorType"
|
||||
}`)))
|
||||
w := httptest.NewRecorder()
|
||||
testEngine.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(1100), returnBody.Code)
|
||||
assert.Equal(t, "vectorFieldType can only be [FloatVector, BinaryVector, Float16Vector, BFloat16Vector, SparseFloatVector], default: FloatVector: invalid parameter[expected=FloatVector, BinaryVector, Float16Vector, BFloat16Vector, SparseFloatVector][actual=InvalidVectorType]", returnBody.Message)
|
||||
assert.Nil(t, proxy.createReq)
|
||||
assert.Empty(t, proxy.createIndexReqs)
|
||||
})
|
||||
|
||||
t.Run("reject sparse vector dimension", func(t *testing.T) {
|
||||
proxy := &externalCollectionRESTProxy{}
|
||||
testEngine := initHTTPServerV2(proxy, false)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(`{
|
||||
"collectionName": "book",
|
||||
"dimension": 4,
|
||||
"vectorFieldType": "SparseFloatVector"
|
||||
}`)))
|
||||
w := httptest.NewRecorder()
|
||||
testEngine.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(1100), returnBody.Code)
|
||||
assert.Contains(t, returnBody.Message, "dimension should not be specified for SparseFloatVector quick create")
|
||||
assert.Nil(t, proxy.createReq)
|
||||
assert.Empty(t, proxy.createIndexReqs)
|
||||
})
|
||||
|
||||
invalidMetricCases := []struct {
|
||||
name string
|
||||
body string
|
||||
expectedMsg string
|
||||
}{
|
||||
{
|
||||
name: "reject binary vector invalid metric",
|
||||
body: `{
|
||||
"collectionName": "book",
|
||||
"dimension": 8,
|
||||
"metricType": "COSINE",
|
||||
"vectorFieldType": "BinaryVector"
|
||||
}`,
|
||||
expectedMsg: "binary vector index does not support metric type: COSINE",
|
||||
},
|
||||
{
|
||||
name: "reject sparse float vector invalid metric",
|
||||
body: `{
|
||||
"collectionName": "book",
|
||||
"metricType": "COSINE",
|
||||
"vectorFieldType": "SparseFloatVector"
|
||||
}`,
|
||||
expectedMsg: "only IP&BM25 is the supported metric type for sparse index",
|
||||
},
|
||||
{
|
||||
name: "reject sparse float vector bm25 metric",
|
||||
body: `{
|
||||
"collectionName": "book",
|
||||
"metricType": "BM25",
|
||||
"vectorFieldType": "SparseFloatVector"
|
||||
}`,
|
||||
expectedMsg: "only BM25 Function output field support BM25 metric type",
|
||||
},
|
||||
}
|
||||
for _, testcase := range invalidMetricCases {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
proxy := &externalCollectionRESTProxy{}
|
||||
testEngine := initHTTPServerV2(proxy, false)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(testcase.body)))
|
||||
w := httptest.NewRecorder()
|
||||
testEngine.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(1100), returnBody.Code)
|
||||
assert.Contains(t, returnBody.Message, testcase.expectedMsg)
|
||||
assert.Nil(t, proxy.createReq)
|
||||
assert.Empty(t, proxy.createIndexReqs)
|
||||
})
|
||||
}
|
||||
|
||||
testcases := []struct {
|
||||
name string
|
||||
body string
|
||||
expectedType schemapb.DataType
|
||||
expectedDimParam bool
|
||||
expectedMetric string
|
||||
}{
|
||||
{
|
||||
name: "binary vector default metric",
|
||||
body: `{
|
||||
"collectionName": "book",
|
||||
"dimension": 8,
|
||||
"vectorFieldType": "BinaryVector"
|
||||
}`,
|
||||
expectedType: schemapb.DataType_BinaryVector,
|
||||
expectedDimParam: true,
|
||||
expectedMetric: paramtable.BinaryVectorDefaultMetricType,
|
||||
},
|
||||
{
|
||||
name: "float16 vector",
|
||||
body: `{
|
||||
"collectionName": "book",
|
||||
"dimension": 4,
|
||||
"vectorFieldType": "Float16Vector"
|
||||
}`,
|
||||
expectedType: schemapb.DataType_Float16Vector,
|
||||
expectedDimParam: true,
|
||||
expectedMetric: DefaultMetricType,
|
||||
},
|
||||
{
|
||||
name: "bfloat16 vector",
|
||||
body: `{
|
||||
"collectionName": "book",
|
||||
"dimension": 4,
|
||||
"vectorFieldType": "BFloat16Vector"
|
||||
}`,
|
||||
expectedType: schemapb.DataType_BFloat16Vector,
|
||||
expectedDimParam: true,
|
||||
expectedMetric: DefaultMetricType,
|
||||
},
|
||||
{
|
||||
name: "sparse float vector default metric",
|
||||
body: `{
|
||||
"collectionName": "book",
|
||||
"vectorFieldType": "SparseFloatVector"
|
||||
}`,
|
||||
expectedType: schemapb.DataType_SparseFloatVector,
|
||||
expectedDimParam: false,
|
||||
expectedMetric: paramtable.SparseFloatVectorDefaultMetricType,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testcase := range testcases {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
proxy := &externalCollectionRESTProxy{}
|
||||
testEngine := initHTTPServerV2(proxy, false)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(testcase.body)))
|
||||
w := httptest.NewRecorder()
|
||||
testEngine.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(0), returnBody.Code)
|
||||
require.NotNil(t, proxy.createReq)
|
||||
|
||||
collSchema := &schemapb.CollectionSchema{}
|
||||
require.NoError(t, proto.Unmarshal(proxy.createReq.GetSchema(), collSchema))
|
||||
require.Len(t, collSchema.GetFields(), 2)
|
||||
vectorField := collSchema.GetFields()[1]
|
||||
assert.Equal(t, testcase.expectedType, vectorField.GetDataType())
|
||||
|
||||
dimFound := false
|
||||
for _, typeParam := range vectorField.GetTypeParams() {
|
||||
if typeParam.GetKey() == Dim {
|
||||
dimFound = true
|
||||
}
|
||||
}
|
||||
assert.Equal(t, testcase.expectedDimParam, dimFound)
|
||||
|
||||
require.Len(t, proxy.createIndexReqs, 1)
|
||||
indexReq := proxy.createIndexReqs[0]
|
||||
assert.Equal(t, DefaultVectorFieldName, indexReq.GetFieldName())
|
||||
assert.Equal(t, DefaultVectorFieldName, indexReq.GetIndexName())
|
||||
require.Len(t, indexReq.GetExtraParams(), 1)
|
||||
assert.Equal(t, common.MetricTypeKey, indexReq.GetExtraParams()[0].GetKey())
|
||||
assert.Equal(t, testcase.expectedMetric, indexReq.GetExtraParams()[0].GetValue())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCollectionTopLevelConsistencyLevel(t *testing.T) {
|
||||
paramtable.Init()
|
||||
paramtable.Get().Save(paramtable.Get().QuotaConfig.QuotaAndLimitsEnabled.Key, "false")
|
||||
defer paramtable.Get().Reset(paramtable.Get().QuotaConfig.QuotaAndLimitsEnabled.Key)
|
||||
|
||||
path := versionalV2(CollectionCategory, CreateAction)
|
||||
|
||||
t.Run("reject invalid top-level consistency level", func(t *testing.T) {
|
||||
proxy := &externalCollectionRESTProxy{}
|
||||
testEngine := initHTTPServerV2(proxy, false)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(`{
|
||||
"collectionName": "book",
|
||||
"dimension": 4,
|
||||
"consistencyLevel": "Invalid"
|
||||
}`)))
|
||||
w := httptest.NewRecorder()
|
||||
testEngine.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(1100), returnBody.Code)
|
||||
assert.Contains(t, returnBody.Message, "consistencyLevel can only be")
|
||||
assert.Nil(t, proxy.createReq)
|
||||
assert.Empty(t, proxy.createIndexReqs)
|
||||
})
|
||||
|
||||
t.Run("accept valid top-level consistency level", func(t *testing.T) {
|
||||
proxy := &externalCollectionRESTProxy{}
|
||||
testEngine := initHTTPServerV2(proxy, false)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(`{
|
||||
"collectionName": "book",
|
||||
"dimension": 4,
|
||||
"consistencyLevel": "Strong"
|
||||
}`)))
|
||||
w := httptest.NewRecorder()
|
||||
testEngine.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(0), returnBody.Code)
|
||||
require.NotNil(t, proxy.createReq)
|
||||
assert.Equal(t, commonpb.ConsistencyLevel_Strong, proxy.createReq.GetConsistencyLevel())
|
||||
require.Len(t, proxy.createIndexReqs, 1)
|
||||
})
|
||||
|
||||
t.Run("reject conflicting consistency levels", func(t *testing.T) {
|
||||
proxy := &externalCollectionRESTProxy{}
|
||||
testEngine := initHTTPServerV2(proxy, false)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(`{
|
||||
"collectionName": "book",
|
||||
"dimension": 4,
|
||||
"consistencyLevel": "Strong",
|
||||
"params": {"consistencyLevel": "Bounded"}
|
||||
}`)))
|
||||
w := httptest.NewRecorder()
|
||||
testEngine.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
returnBody := &ReturnErrMsg{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), returnBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int32(1100), returnBody.Code)
|
||||
assert.Contains(t, returnBody.Message, "top-level consistencyLevel conflicts with params.consistencyLevel")
|
||||
assert.Nil(t, proxy.createReq)
|
||||
assert.Empty(t, proxy.createIndexReqs)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCreateCollectionStructArrayDuplicateName(t *testing.T) {
|
||||
paramtable.Init()
|
||||
paramtable.Get().Save(paramtable.Get().QuotaConfig.QuotaAndLimitsEnabled.Key, "false")
|
||||
@@ -2110,6 +2376,7 @@ func initHTTPServerV2(proxy types.ProxyComponent, needAuth bool) *gin.Engine {
|
||||
type externalCollectionRESTProxy struct {
|
||||
mockProxyComponent
|
||||
createReq *milvuspb.CreateCollectionRequest
|
||||
createIndexReqs []*milvuspb.CreateIndexRequest
|
||||
describeResp *milvuspb.DescribeCollectionResponse
|
||||
refreshReq *milvuspb.RefreshExternalCollectionRequest
|
||||
listReq *milvuspb.ListRefreshExternalCollectionJobsRequest
|
||||
@@ -2125,6 +2392,11 @@ func (m *externalCollectionRESTProxy) CreateCollection(ctx context.Context, requ
|
||||
return commonSuccessStatus, nil
|
||||
}
|
||||
|
||||
func (m *externalCollectionRESTProxy) CreateIndex(ctx context.Context, request *milvuspb.CreateIndexRequest) (*commonpb.Status, error) {
|
||||
m.createIndexReqs = append(m.createIndexReqs, request)
|
||||
return commonSuccessStatus, nil
|
||||
}
|
||||
|
||||
func (m *externalCollectionRESTProxy) DescribeCollection(ctx context.Context, request *milvuspb.DescribeCollectionRequest) (*milvuspb.DescribeCollectionResponse, error) {
|
||||
if m.describeResp != nil {
|
||||
return m.describeResp, nil
|
||||
|
||||
@@ -939,6 +939,8 @@ type CollectionReq struct {
|
||||
IDType string `json:"idType"`
|
||||
AutoID bool `json:"autoID"`
|
||||
MetricType string `json:"metricType"`
|
||||
VectorFieldType string `json:"vectorFieldType"`
|
||||
ConsistencyLevel string `json:"consistencyLevel"`
|
||||
PrimaryFieldName string `json:"primaryFieldName"`
|
||||
VectorFieldName string `json:"vectorFieldName"`
|
||||
Schema CollectionSchema `json:"schema"`
|
||||
|
||||
@@ -82,6 +82,45 @@ class TestCreateCollection(TestBase):
|
||||
for index in rsp["data"]["indexes"]:
|
||||
assert index["metricType"] == metric_type
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_field_type,dimension,expected_metric",
|
||||
[
|
||||
("Float16Vector", 4, "COSINE"),
|
||||
("BFloat16Vector", 4, "COSINE"),
|
||||
("BinaryVector", 8, "HAMMING"),
|
||||
("SparseFloatVector", None, "IP"),
|
||||
],
|
||||
)
|
||||
def test_create_collection_quick_setup_vector_field_type_default_metric(
|
||||
self, vector_field_type, dimension, expected_metric
|
||||
):
|
||||
"""
|
||||
target: test quick create collection with vectorFieldType and default metricType
|
||||
method: create collection with BinaryVector/SparseFloatVector without metricType
|
||||
expected: create collection succeeds with compatible default index metric
|
||||
"""
|
||||
name = gen_collection_name()
|
||||
client = self.collection_client
|
||||
payload = {
|
||||
"collectionName": name,
|
||||
"vectorFieldType": vector_field_type,
|
||||
}
|
||||
if dimension is not None:
|
||||
payload["dimension"] = dimension
|
||||
|
||||
logging.info(f"create collection {name} with payload: {payload}")
|
||||
rsp = client.collection_create(payload)
|
||||
assert rsp["code"] == 0
|
||||
|
||||
rsp = client.collection_describe(name)
|
||||
assert rsp["code"] == 0
|
||||
assert rsp["data"]["collectionName"] == name
|
||||
vector_fields = [field for field in rsp["data"]["fields"] if field["name"] == "vector"]
|
||||
assert len(vector_fields) == 1
|
||||
assert vector_fields[0]["type"] == vector_field_type
|
||||
assert len(rsp["data"]["indexes"]) == 1
|
||||
assert rsp["data"]["indexes"][0]["metricType"] == expected_metric
|
||||
|
||||
@pytest.mark.parametrize("enable_dynamic_field", [False, "False", "0"])
|
||||
@pytest.mark.parametrize("request_shards_num", [2, "2"])
|
||||
@pytest.mark.parametrize("request_ttl_seconds", [360, "360"])
|
||||
@@ -811,6 +850,106 @@ class TestCreateCollectionNegative(TestBase):
|
||||
rsp = client.collection_create(payload)
|
||||
assert rsp["code"] == 1801
|
||||
|
||||
def test_create_collection_quick_setup_with_invalid_vector_field_type(self):
|
||||
"""
|
||||
target: test quick create collection with invalid vectorFieldType
|
||||
method: create collection with invalid vectorFieldType
|
||||
expected: create collection failed with right error message
|
||||
"""
|
||||
name = gen_collection_name()
|
||||
client = self.collection_client
|
||||
payload = {
|
||||
"collectionName": name,
|
||||
"dimension": 4,
|
||||
"metricType": "L2",
|
||||
"idType": "Int64",
|
||||
"autoID": True,
|
||||
"vectorFieldType": "InvalidVectorType",
|
||||
}
|
||||
logging.info(f"create collection {name} with payload: {payload}")
|
||||
rsp = client.collection_create(payload)
|
||||
assert rsp["code"] == 1100
|
||||
assert "vectorFieldType can only be" in rsp["message"]
|
||||
|
||||
rsp = client.collection_list()
|
||||
assert name not in rsp["data"]
|
||||
|
||||
def test_create_collection_quick_setup_with_invalid_consistency_level(self):
|
||||
"""
|
||||
target: test quick create collection with invalid top-level consistencyLevel
|
||||
method: create collection with invalid consistencyLevel
|
||||
expected: create collection failed with right error message
|
||||
"""
|
||||
name = gen_collection_name()
|
||||
client = self.collection_client
|
||||
payload = {
|
||||
"collectionName": name,
|
||||
"dimension": 4,
|
||||
"consistencyLevel": "Invalid",
|
||||
}
|
||||
logging.info(f"create collection {name} with payload: {payload}")
|
||||
url = f"{client.endpoint}/v2/vectordb/collections/create"
|
||||
rsp = client.post(url, headers=client.update_headers(), data=payload).json()
|
||||
assert rsp["code"] == 1100
|
||||
assert "consistencyLevel can only be" in rsp["message"]
|
||||
|
||||
rsp = client.collection_list()
|
||||
assert name not in rsp["data"]
|
||||
|
||||
def test_create_collection_quick_setup_sparse_vector_with_dimension(self):
|
||||
"""
|
||||
target: test quick create collection with SparseFloatVector and dimension
|
||||
method: create collection with SparseFloatVector and dimension
|
||||
expected: create collection failed with right error message
|
||||
"""
|
||||
name = gen_collection_name()
|
||||
client = self.collection_client
|
||||
payload = {
|
||||
"collectionName": name,
|
||||
"dimension": 4,
|
||||
"vectorFieldType": "SparseFloatVector",
|
||||
}
|
||||
logging.info(f"create collection {name} with payload: {payload}")
|
||||
rsp = client.collection_create(payload)
|
||||
assert rsp["code"] == 1100
|
||||
assert "dimension should not be specified for SparseFloatVector quick create" in rsp["message"]
|
||||
|
||||
rsp = client.collection_list()
|
||||
assert name not in rsp["data"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_field_type,dimension,metric_type,expected_message",
|
||||
[
|
||||
("BinaryVector", 8, "COSINE", "binary vector index does not support metric type: COSINE"),
|
||||
("SparseFloatVector", None, "COSINE", "only IP&BM25 is the supported metric type for sparse index"),
|
||||
("SparseFloatVector", None, "BM25", "only BM25 Function output field support BM25 metric type"),
|
||||
],
|
||||
)
|
||||
def test_create_collection_quick_setup_vector_field_type_invalid_metric(
|
||||
self, vector_field_type, dimension, metric_type, expected_message
|
||||
):
|
||||
"""
|
||||
target: test quick create collection with vectorFieldType and invalid metricType
|
||||
method: create collection with BinaryVector/SparseFloatVector and incompatible metricType
|
||||
expected: create collection failed before collection is created
|
||||
"""
|
||||
name = gen_collection_name()
|
||||
client = self.collection_client
|
||||
payload = {
|
||||
"collectionName": name,
|
||||
"metricType": metric_type,
|
||||
"vectorFieldType": vector_field_type,
|
||||
}
|
||||
if dimension is not None:
|
||||
payload["dimension"] = dimension
|
||||
logging.info(f"create collection {name} with payload: {payload}")
|
||||
rsp = client.collection_create(payload)
|
||||
assert rsp["code"] == 1100
|
||||
assert expected_message in rsp["message"]
|
||||
|
||||
rsp = client.collection_list()
|
||||
assert name not in rsp["data"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name", [" ", "test_collection_" * 100, "test collection", "test/collection", r"test\collection"]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user