enhance: update default auto index for vector filed (#47387)

issue: #47386 
/kind improvement

Signed-off-by: xianliang.li <xianliang.li@zilliz.com>

internal/proxy/task_index.go

@@ -53,8 +53,51 @@ const (
 	AutoIndexName = common.AutoIndexName
 	DimKey        = common.DimKey
 	IsSparseKey   = common.IsSparseKey
+
+	RefineTypeKey = "refine_type"
 )
 
+// adjustAutoIndexParamsByDataType adjusts autoindex params based on vector data type
+// If data_type is bf16 and refine_type is fp16/fp32, adjust to BF16
+// If data_type is fp16 and refine_type is bf16/fp32, adjust to FP16
+// Other refine_type values (e.g., sq8) are not modified
+func adjustAutoIndexParamsByDataType(config map[string]string, dataType schemapb.DataType) map[string]string {
+	if config == nil {
+		return config
+	}
+	refineType, hasRefine := config[RefineTypeKey]
+	if !hasRefine {
+		return config
+	}
+
+	refineTypeLower := strings.ToLower(refineType)
+	var requiredRefineType string
+
+	switch dataType {
+	case schemapb.DataType_Float16Vector:
+		// fp16 data requires fp16 refine, adjust if refine_type is bf16/fp32
+		if refineTypeLower == "bf16" || refineTypeLower == "fp32" {
+			requiredRefineType = "FP16"
+		}
+	case schemapb.DataType_BFloat16Vector:
+		// bf16 data requires bf16 refine, adjust if refine_type is fp16/fp32
+		if refineTypeLower == "fp16" || refineTypeLower == "fp32" {
+			requiredRefineType = "BF16"
+		}
+	}
+
+	if requiredRefineType == "" {
+		return config
+	}
+
+	adjusted := make(map[string]string, len(config))
+	for k, v := range config {
+		adjusted[k] = v
+	}
+	adjusted[RefineTypeKey] = requiredRefineType
+	return adjusted
+}
+
 type createIndexTask struct {
 	baseTask
 	Condition
@@ -265,7 +308,9 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
 
 			if typeutil.IsDenseFloatVectorType(cit.fieldSchema.DataType) {
 				// override float vector index params by autoindex
-				for k, v := range Params.AutoIndexConfig.IndexParams.GetAsJSONMap() {
+				// filter incompatible refine_type for fp16/bf16 vectors
+				autoIndexParams := adjustAutoIndexParamsByDataType(Params.AutoIndexConfig.IndexParams.GetAsJSONMap(), cit.fieldSchema.DataType)
+				for k, v := range autoIndexParams {
 					indexParamsMap[k] = v
 				}
 			} else if typeutil.IsSparseFloatVectorType(cit.fieldSchema.DataType) {
@@ -346,7 +391,12 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
 			if typeutil.IsDenseFloatVectorType(cit.fieldSchema.DataType) ||
 				(typeutil.IsArrayOfVectorType(cit.fieldSchema.DataType) && typeutil.IsDenseFloatVectorType(cit.fieldSchema.ElementType)) {
 				// override float vector index params by autoindex
-				config = Params.AutoIndexConfig.IndexParams.GetAsJSONMap()
+				// filter incompatible refine_type for fp16/bf16 vectors
+				dataType := cit.fieldSchema.DataType
+				if typeutil.IsArrayOfVectorType(cit.fieldSchema.DataType) {
+					dataType = cit.fieldSchema.ElementType
+				}
+				config = adjustAutoIndexParamsByDataType(Params.AutoIndexConfig.IndexParams.GetAsJSONMap(), dataType)
 			} else if typeutil.IsSparseFloatVectorType(cit.fieldSchema.DataType) ||
 				(typeutil.IsArrayOfVectorType(cit.fieldSchema.DataType) && typeutil.IsSparseFloatVectorType(cit.fieldSchema.ElementType)) {
 				// override sparse float vector index params by autoindex

internal/proxy/task_index_test.go

@@ -1631,3 +1631,103 @@ func newTestSchema() *schemapb.CollectionSchema {
 		EnableDynamicField: true,
 	}
 }
+
+func TestAdjustAutoIndexParamsByDataType(t *testing.T) {
+	t.Run("nil config", func(t *testing.T) {
+		result := adjustAutoIndexParamsByDataType(nil, schemapb.DataType_FloatVector)
+		assert.Nil(t, result)
+	})
+
+	t.Run("no refine_type", func(t *testing.T) {
+		config := map[string]string{
+			"index_type": "HNSW_SQ",
+			"M":          "18",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_BFloat16Vector)
+		assert.Equal(t, config, result)
+	})
+
+	t.Run("fp32 vector keeps any refine_type", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "FP16",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_FloatVector)
+		assert.Contains(t, result, "refine_type")
+		assert.Equal(t, "FP16", result["refine_type"])
+	})
+
+	t.Run("fp16 vector keeps fp16 refine_type", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "FP16",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_Float16Vector)
+		assert.Equal(t, config, result) // same object, no copy needed
+	})
+
+	t.Run("fp16 vector adjusts bf16 refine_type to FP16", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "BF16",
+			"M":           "18",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_Float16Vector)
+		assert.Contains(t, result, "refine_type")
+		assert.Equal(t, "FP16", result["refine_type"])
+		assert.Contains(t, result, "index_type")
+		assert.Contains(t, result, "M")
+	})
+
+	t.Run("bf16 vector keeps bf16 refine_type", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "BF16",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_BFloat16Vector)
+		assert.Equal(t, config, result) // same object, no copy needed
+	})
+
+	t.Run("bf16 vector adjusts fp16 refine_type to BF16", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "FP16",
+			"M":           "18",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_BFloat16Vector)
+		assert.Contains(t, result, "refine_type")
+		assert.Equal(t, "BF16", result["refine_type"])
+		assert.Contains(t, result, "index_type")
+		assert.Contains(t, result, "M")
+	})
+
+	t.Run("fp16 vector keeps other refine_type unchanged", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "SQ8",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_Float16Vector)
+		assert.Equal(t, config, result) // not modified
+	})
+
+	t.Run("bf16 vector keeps other refine_type unchanged", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "SQ8",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_BFloat16Vector)
+		assert.Equal(t, config, result) // not modified
+	})
+
+	t.Run("original config not modified", func(t *testing.T) {
+		config := map[string]string{
+			"index_type":  "HNSW_SQ",
+			"refine_type": "FP16",
+		}
+		result := adjustAutoIndexParamsByDataType(config, schemapb.DataType_BFloat16Vector)
+		// Original config should still have original refine_type
+		assert.Equal(t, "FP16", config["refine_type"])
+		// Result should have replaced refine_type
+		assert.Equal(t, "BF16", result["refine_type"])
+	})
+}

pkg/util/paramtable/autoindex_param_nocuda.go

@@ -23,7 +23,7 @@ func GetIndexParam() ParamItem {
 	return ParamItem{
 		Key:          "autoIndex.params.build",
 		Version:      "2.2.0",
-		DefaultValue: `{"M": 18,"efConstruction": 240,"index_type": "HNSW", "metric_type": "COSINE"}`,
+		DefaultValue: `{"M": 18 ,"efConstruction": 240,"index_type": "HNSW_SQ", "sq_type": "SQ4U", "refine": true, "refine_type": "FP16", "metric_type": "COSINE"}`,
 		Formatter:    GetBuildParamFormatter(FloatVectorDefaultMetricType, "autoIndex.params.build"),
 		Export:       true,
 	}

pkg/util/paramtable/autoindex_param_test.go

@@ -185,3 +185,42 @@ func TestScalarAutoIndexParams_build(t *testing.T) {
 		assert.Equal(t, "INVERTED", CParams.AutoIndexConfig.ScalarBoolIndexType.GetValue())
 	})
 }
+
+func TestGetIndexParam_DefaultValue(t *testing.T) {
+	paramItem := GetIndexParam()
+
+	t.Run("default value is valid JSON", func(t *testing.T) {
+		var params map[string]any
+		err := json.Unmarshal([]byte(paramItem.DefaultValue), &params)
+		assert.NoError(t, err, "DefaultValue should be valid JSON")
+	})
+
+	t.Run("default value contains required fields", func(t *testing.T) {
+		var params map[string]any
+		err := json.Unmarshal([]byte(paramItem.DefaultValue), &params)
+		assert.NoError(t, err)
+
+		assert.Contains(t, params, IndexTypeKey, "DefaultValue should contain index_type")
+		assert.Contains(t, params, MetricTypeKey, "DefaultValue should contain metric_type")
+	})
+
+	t.Run("default value can be parsed by config", func(t *testing.T) {
+		mgr := config.NewManager()
+		mgr.SetConfig(paramItem.Key, paramItem.DefaultValue)
+
+		p := &AutoIndexConfig{
+			IndexParams: ParamItem{
+				Key:       paramItem.Key,
+				Formatter: GetBuildParamFormatter(FloatVectorDefaultMetricType, paramItem.Key),
+			},
+		}
+
+		assert.NotPanics(t, func() {
+			p.IndexParams.Init(mgr)
+		}, "DefaultValue should be parseable without panic")
+
+		jsonMap := p.IndexParams.GetAsJSONMap()
+		assert.NotEmpty(t, jsonMap[IndexTypeKey], "index_type should not be empty")
+		assert.NotEmpty(t, jsonMap[MetricTypeKey], "metric_type should not be empty")
+	})
+}