furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: add utf8 check before bm25 functoin run (#44220)

Browse Source 
relate: https://github.com/milvus-io/milvus/issues/44219

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
aoiasd
2025-09-08 14:43:56 +08:00
committed by GitHub
parent 083f4e0371
commit f98dd84dec
2 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/util/function/bm25_function.go
+4
View File
@@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/util/analyzer"
"github.com/milvus-io/milvus/pkg/v2/util/merr"
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
)
@@ -109,6 +110,9 @@ func (v *BM25FunctionRunner) run(data []string, dst []map[uint32]float32) error
defer tokenizer.Destroy()
for i := 0; i < len(data); i++ {
if !typeutil.IsUTF8(data[i]) {
return merr.WrapErrParameterInvalidMsg("string data must be utf8 format: %v", data[i])
}
embeddingMap := map[uint32]float32{}
tokenStream := tokenizer.NewTokenStream(data[i])
defer tokenStream.Destroy()
internal/util/function/multi_analyzer_bm25_function.go
+4
View File
@@ -26,6 +26,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/util/analyzer"
"github.com/milvus-io/milvus/pkg/v2/util/merr"
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
)
@@ -153,6 +154,9 @@ func (v *MultiAnalyzerBM25FunctionRunner) run(text []string, analyzerName []stri
}()
for i := 0; i < len(text); i++ {
if !typeutil.IsUTF8(text[i]) {
return merr.WrapErrParameterInvalidMsg("string data must be utf8 format: %v", text[i])
}
embeddingMap := map[uint32]float32{}
analyzer, err := v.getAnalyzer(analyzerName[i], cloneAnalyzers)