furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 02:05:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix ngram index UTF-8 literal length checks (#51238)

Browse Source 
issue: https://github.com/milvus-io/milvus/issues/42053

This PR fixes ngram index eligibility checks for UTF-8 literals by using
character count instead of byte length in the C++ gate and phase1
validation. It also adds regression coverage for short multibyte
literals to ensure they correctly fall back instead of being sent to
Tantivy ngram queries.

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
This commit is contained in:
Spade A
2026-07-14 16:38:36 +08:00
committed by GitHub
parent 69da6f0f66
commit 9e11bdae4e
2 changed files with 85 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/core/src/index/NgramInvertedIndex.cpp
+15 -10
View File
@@ -75,6 +75,11 @@ constexpr size_t kMaxIterationsForMediumRow = 3;
constexpr double kBreakThresholdForSmallRow = 0.01; // 1%
constexpr size_t kMaxIterationsForSmallRow = 2;
inline size_t
Utf8LiteralLength(const std::string& literal) {
return Utf8CharCount(literal.data(), literal.size());
}
// for string/varchar type
NgramInvertedIndex::NgramInvertedIndex(const storage::FileManagerContext& ctx,
const NgramParams& params)
@@ -804,7 +809,7 @@ NgramInvertedIndex::CanHandleLiteral(const std::string& literal,
return false;
}
for (const auto& l : literals) {
if (l.length() < min_gram_) {
if (Utf8LiteralLength(l) < min_gram_) {
return false;
}
}
@@ -816,7 +821,7 @@ NgramInvertedIndex::CanHandleLiteral(const std::string& literal,
return false;
}
for (const auto& l : literals) {
if (l.length() >= min_gram_) {
if (Utf8LiteralLength(l) >= min_gram_) {
return true;
}
}
@@ -825,7 +830,7 @@ NgramInvertedIndex::CanHandleLiteral(const std::string& literal,
case proto::plan::OpType::InnerMatch:
case proto::plan::OpType::PrefixMatch:
case proto::plan::OpType::PostfixMatch:
return literal.length() >= min_gram_;
return Utf8LiteralLength(literal) >= min_gram_;
default:
return false;
}
@@ -924,16 +929,16 @@ NgramInvertedIndex::ExecutePhase1(const std::string& literal,
AssertInfo(!literals_vec.empty(),
"ExecutePhase1: Match pattern must have non-empty parts");
for (const auto& l : literals_vec) {
AssertInfo(l.length() >= min_gram_,
"ExecutePhase1: part length {} < min_gram {}",
l.length(),
AssertInfo(Utf8LiteralLength(l) >= min_gram_,
"ExecutePhase1: part char length {} < min_gram {}",
Utf8LiteralLength(l),
min_gram_);
}
} else if (op_type == proto::plan::OpType::RegexMatch) {
auto all_literals = extract_literals_from_regex(literal);
// Only keep literals that are long enough for ngram
for (const auto& l : all_literals) {
if (l.length() >= min_gram_) {
if (Utf8LiteralLength(l) >= min_gram_) {
literals_vec.push_back(l);
}
}
@@ -941,9 +946,9 @@ NgramInvertedIndex::ExecutePhase1(const std::string& literal,
"ExecutePhase1: RegexMatch pattern must have non-empty "
"literals >= min_gram");
} else {
AssertInfo(literal.length() >= min_gram_,
"ExecutePhase1: literal length {} < min_gram {}",
literal.length(),
AssertInfo(Utf8LiteralLength(literal) >= min_gram_,
"ExecutePhase1: literal char length {} < min_gram {}",
Utf8LiteralLength(literal),
min_gram_);
literals_vec.push_back(literal);
}
internal/core/src/index/NgramInvertedIndexTest.cpp
+70
View File
@@ -89,6 +89,38 @@ using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus::exec;
std::unique_ptr<index::NgramInvertedIndex>
CreateNgramIndexForCanHandleLiteral(uintptr_t min_gram = 2,
uintptr_t max_gram = 4) {
int64_t collection_id = 1;
int64_t partition_id = 2;
int64_t segment_id = 3;
int64_t index_build_id = 4000;
int64_t index_version = 4000;
auto schema = std::make_shared<Schema>();
auto field_id = schema->AddDebugField("ngram", DataType::VARCHAR);
auto field_meta = milvus::segcore::gen_field_meta(collection_id,
partition_id,
segment_id,
field_id.get(),
DataType::VARCHAR,
DataType::NONE,
false);
auto index_meta = gen_index_meta(
segment_id, field_id.get(), index_build_id, index_version);
auto storage_config = gen_local_storage_config(TestLocalPath);
auto cm = CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
auto ngram_params = index::NgramParams{
.loading_index = true, .min_gram = min_gram, .max_gram = max_gram};
return std::make_unique<index::NgramInvertedIndex>(ctx, ngram_params);
}
void
test_ngram_with_data(const boost::container::vector<std::string>& data,
const std::string& literal,
@@ -1499,6 +1531,44 @@ TEST(NgramPatternMatchConsistency, UTF8Patterns) {
}
}
TEST(NgramPatternMatchConsistency, CanHandleLiteralUsesUtf8CharacterCount) {
auto ngram_index = CreateNgramIndexForCanHandleLiteral(2, 4);
const std::string single_chinese = "测";
const std::string two_chinese = "测试";
ASSERT_EQ(single_chinese.size(), 3);
ASSERT_EQ(Utf8CharCount(single_chinese.data(), single_chinese.size()), 1);
ASSERT_EQ(Utf8CharCount(two_chinese.data(), two_chinese.size()), 2);
struct TestCase {
std::string literal;
proto::plan::OpType op_type;
bool expected_can_handle;
};
std::vector<TestCase> test_cases = {
{single_chinese, proto::plan::OpType::InnerMatch, false},
{single_chinese, proto::plan::OpType::PrefixMatch, false},
{single_chinese, proto::plan::OpType::PostfixMatch, false},
{"%" + single_chinese + "%", proto::plan::OpType::Match, false},
{single_chinese, proto::plan::OpType::RegexMatch, false},
{two_chinese, proto::plan::OpType::InnerMatch, true},
{"%" + two_chinese + "%", proto::plan::OpType::Match, true},
{two_chinese, proto::plan::OpType::RegexMatch, true},
};
for (const auto& test_case : test_cases) {
EXPECT_EQ(
ngram_index->CanHandleLiteral(test_case.literal, test_case.op_type),
test_case.expected_can_handle)
<< "literal=\"" << test_case.literal << "\""
<< ", op_type=" << static_cast<int>(test_case.op_type)
<< ", byte_len=" << test_case.literal.size() << ", utf8_char_count="
<< Utf8CharCount(test_case.literal.data(),
test_case.literal.size());
}
}
TEST(NgramPatternMatchConsistency, EscapeSequences) {
// Test data with special characters
boost::container::vector<std::string> test_data = {