furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance: refine inverted pattern match planner policy (#49922)

Browse Source 
issue: #50005

## Summary
Separate pattern match capability from planner policy so inverted
indexes keep regex filters on the index path while suffix/contains LIKE
can fall back to raw-data scan when appropriate.

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
aoiasd
2026-06-01 20:08:16 +08:00
committed by GitHub
parent 99cf5e8d48
commit ccd649b2fc
12 changed files with 212 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/core/src/common/RegexQueryTest.cpp
-5
View File
@@ -212,11 +212,6 @@ struct MockStringIndex : index::StringIndexMarisa {
HasRawData() const override {
return true;
}
bool
SupportPatternQuery() const override {
return false;
}
};
class SealedSegmentRegexQueryTest : public ::testing::Test {
internal/core/src/exec/expression/Expr.h
+6 -17
View File
@@ -2049,23 +2049,12 @@ class SegmentExpr : public Expr {
}
using Index = index::ScalarIndex<IndexInnerType>;
if (op == OpType::Match || op == OpType::InnerMatch ||
op == OpType::PostfixMatch || op == OpType::RegexMatch) {
AssertInfo(num_index_chunk_ == 1,
"scalar index should have exactly 1 chunk, got {}",
num_index_chunk_);
auto scalar_index =
dynamic_cast<const Index*>(pinned_index_[0].get());
auto* index_ptr = const_cast<Index*>(scalar_index);
// 1. index supports PatternMatch (iterates unique values) — preferred;
// 2. index supports PatternQuery via TryUsePatternQuery;
// 3. index has raw data for Reverse_Lookup fallback.
return index_ptr->SupportPatternMatch() ||
(index_ptr->TryUsePatternQuery() &&
index_ptr->SupportPatternQuery()) ||
index_ptr->HasRawData();
}
return true;
AssertInfo(num_index_chunk_ == 1,
"scalar index should have exactly 1 chunk, got {}",
num_index_chunk_);
auto scalar_index = dynamic_cast<const Index*>(pinned_index_[0].get());
AssertInfo(scalar_index != nullptr, "invalid scalar index type");
return scalar_index->ShouldUseOp(op);
}
template <typename T>
internal/core/src/index/BitmapIndex.h
+39 -43
View File
@@ -207,49 +207,6 @@ class BitmapIndex : public ScalarIndex<T> {
return std::is_same_v<T, std::string>;
}
bool
SupportPatternQuery() const override {
return std::is_same_v<T, std::string>;
}
const TargetBitmap
PatternQuery(const std::string& pattern) override {
if constexpr (!std::is_same_v<T, std::string>) {
ThrowInfo(ErrorCode::OpTypeInvalid,
"pattern query only supported for string type");
return TargetBitmap{};
} else {
AssertInfo(is_built_, "index has not been built");
LikePatternMatcher matcher(pattern);
TargetBitmap res(total_num_rows_, false);
if (is_mmap_) {
for (const auto& [key, bitmap] : bitmap_info_map_) {
if (matcher(key)) {
for (const auto& v : bitmap) {
res.set(v);
}
}
}
} else if (build_mode_ == BitmapIndexBuildMode::ROARING) {
for (const auto& [key, bitmap] : data_) {
if (matcher(key)) {
for (const auto& v : bitmap) {
res.set(v);
}
}
}
} else {
for (const auto& [key, bitset] : bitsets_) {
if (matcher(key)) {
res |= bitset;
}
}
}
return res;
}
}
const TargetBitmap
PatternMatch(const std::string& pattern, proto::plan::OpType op) override {
switch (op) {
@@ -306,6 +263,45 @@ class BitmapIndex : public ScalarIndex<T> {
}
}
protected:
const TargetBitmap
PatternQuery(const std::string& pattern) override {
if constexpr (!std::is_same_v<T, std::string>) {
ThrowInfo(ErrorCode::OpTypeInvalid,
"pattern query only supported for string type");
return TargetBitmap{};
}
AssertInfo(is_built_, "index has not been built");
LikePatternMatcher matcher(pattern);
TargetBitmap res(total_num_rows_, false);
if (is_mmap_) {
for (const auto& [key, bitmap] : bitmap_info_map_) {
if (matcher(key)) {
for (const auto& v : bitmap) {
res.set(v);
}
}
}
} else if (build_mode_ == BitmapIndexBuildMode::ROARING) {
for (const auto& [key, bitmap] : data_) {
if (matcher(key)) {
for (const auto& v : bitmap) {
res.set(v);
}
}
}
} else {
for (const auto& [key, bitset] : bitsets_) {
if (matcher(key)) {
res |= bitset;
}
}
}
return res;
}
public:
int64_t
Cardinality() {
internal/core/src/index/BitmapIndexTest.cpp
+50 -3
View File
@@ -524,6 +524,38 @@ class BitmapIndexTest : public testing::Test {
}
}
void
TestPatternMatchFunc() {
if constexpr (std::is_same_v<T, std::string>) {
auto index_ptr = dynamic_cast<index::BitmapIndex<T>*>(index_.get());
auto like_bitset =
index_ptr->PatternMatch("1%", proto::plan::OpType::Match);
auto regex_bitset = index_ptr->PatternMatch(
"^1.*", proto::plan::OpType::RegexMatch);
ASSERT_EQ(like_bitset.size(), index_ptr->Count());
ASSERT_EQ(regex_bitset.size(), index_ptr->Count());
size_t start = 0;
if (has_lack_binlog_row_) {
for (int i = 0; i < lack_binlog_row_; i++) {
ASSERT_FALSE(like_bitset[i]);
ASSERT_FALSE(regex_bitset[i]);
}
start += lack_binlog_row_;
}
for (size_t i = start; i < like_bitset.size(); i++) {
auto data_offset = i - start;
auto expected = data_[data_offset].rfind("1", 0) == 0;
if (nullable_ && !valid_data_[data_offset]) {
expected = false;
}
ASSERT_EQ(like_bitset[i], expected);
ASSERT_EQ(regex_bitset[i], expected);
}
}
}
public:
IndexBasePtr index_;
DataType type_;
@@ -569,6 +601,10 @@ TYPED_TEST_P(BitmapIndexTest, IsNotNullFuncTest) {
this->TestIsNotNullFunc();
}
TYPED_TEST_P(BitmapIndexTest, PatternMatchFuncTest) {
this->TestPatternMatchFunc();
}
using BitmapType =
testing::Types<int8_t, int16_t, int32_t, int64_t, std::string>;
@@ -578,7 +614,8 @@ REGISTER_TYPED_TEST_SUITE_P(BitmapIndexTest,
NotINFuncTest,
CompareValFuncTest,
IsNullFuncTest,
IsNotNullFuncTest);
IsNotNullFuncTest,
PatternMatchFuncTest);
INSTANTIATE_TYPED_TEST_SUITE_P(BitmapE2ECheck, BitmapIndexTest, BitmapType);
@@ -629,6 +666,10 @@ TYPED_TEST_P(BitmapIndexTestV2, IsNotNullFuncTest) {
this->TestIsNotNullFunc();
}
TYPED_TEST_P(BitmapIndexTestV2, PatternMatchFuncTest) {
this->TestPatternMatchFunc();
}
using BitmapType =
testing::Types<int8_t, int16_t, int32_t, int64_t, std::string>;
@@ -639,7 +680,8 @@ REGISTER_TYPED_TEST_SUITE_P(BitmapIndexTestV2,
CompareValFuncTest,
TestRangeCompareFuncTest,
IsNullFuncTest,
IsNotNullFuncTest);
IsNotNullFuncTest,
PatternMatchFuncTest);
INSTANTIATE_TYPED_TEST_SUITE_P(BitmapIndexE2ECheck_HighCardinality,
BitmapIndexTestV2,
@@ -693,6 +735,10 @@ TYPED_TEST_P(BitmapIndexTestV3, IsNotNullFuncTest) {
this->TestIsNotNullFunc();
}
TYPED_TEST_P(BitmapIndexTestV3, PatternMatchFuncTest) {
this->TestPatternMatchFunc();
}
using BitmapType =
testing::Types<int8_t, int16_t, int32_t, int64_t, std::string>;
@@ -703,7 +749,8 @@ REGISTER_TYPED_TEST_SUITE_P(BitmapIndexTestV3,
CompareValFuncTest,
TestRangeCompareFuncTest,
IsNullFuncTest,
IsNotNullFuncTest);
IsNotNullFuncTest,
PatternMatchFuncTest);
INSTANTIATE_TYPED_TEST_SUITE_P(BitmapIndexE2ECheck_Mmap,
BitmapIndexTestV3,
internal/core/src/index/HybridScalarIndex.h
+6 -16
View File
@@ -110,6 +110,11 @@ class HybridScalarIndex : public ScalarIndex<T> {
return internal_index_->Query(dataset);
}
bool
ShouldUseOp(proto::plan::OpType op) const override {
return internal_index_->ShouldUseOp(op);
}
bool
SupportPatternMatch() const override {
return internal_index_->SupportPatternMatch();
@@ -120,21 +125,6 @@ class HybridScalarIndex : public ScalarIndex<T> {
return internal_index_->PatternMatch(pattern, op);
}
bool
TryUsePatternQuery() const override {
return internal_index_->TryUsePatternQuery();
}
bool
SupportPatternQuery() const override {
return internal_index_->SupportPatternQuery();
}
const TargetBitmap
PatternQuery(const std::string& pattern) override {
return internal_index_->PatternQuery(pattern);
}
const TargetBitmap
Range(const T& value, OpType op) override {
return internal_index_->Range(value, op);
@@ -239,4 +229,4 @@ class HybridScalarIndex : public ScalarIndex<T> {
};
} // namespace index
} // namespace milvus
} // namespace milvus
internal/core/src/index/HybridScalarIndexTest.cpp
+16
View File
@@ -56,6 +56,22 @@ using namespace milvus::indexbuilder;
using namespace milvus;
using namespace milvus::index;
TEST(HybridScalarIndexPlannerPolicy, ShouldUseOpDelegatesToInternalIndex) {
HybridScalarIndex<int64_t> int_index(7);
std::vector<int64_t> int_data{1, 2, 3, 4};
int_index.Build(int_data.size(), int_data.data());
EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::Match));
EXPECT_TRUE(int_index.ShouldUseOp(proto::plan::OpType::Equal));
HybridScalarIndex<std::string> string_index(7);
std::vector<std::string> string_data{"alpha", "beta", "alphabet"};
string_index.Build(string_data.size(), string_data.data());
EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::Match));
EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::PrefixMatch));
EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::RegexMatch));
EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::Equal));
}
template <typename T>
static std::vector<T>
GenerateData(const size_t size, const size_t cardinality) {
internal/core/src/index/InvertedIndexTantivy.h
+18 -11
View File
@@ -284,19 +284,23 @@ class InvertedIndexTantivy : public ScalarIndex<T> {
}
bool
SupportPatternQuery() const override {
return std::is_same_v<T, std::string>;
ShouldUseOp(proto::plan::OpType op) const override {
// Suffix/contains LIKE and regex can be executed correctly over indexed
// terms, but for short varchar values they are often slower than raw
// scan. Keep only prefix/LIKE Match on the inverted-index path.
switch (op) {
case proto::plan::OpType::Match:
case proto::plan::OpType::PrefixMatch:
return SupportPatternMatch() || HasRawData();
case proto::plan::OpType::RegexMatch:
case proto::plan::OpType::PostfixMatch:
case proto::plan::OpType::InnerMatch:
return false;
default:
return true;
}
}
bool
TryUsePatternQuery() const override {
// for inverted index, not use pattern query to implement match
return false;
}
const TargetBitmap
PatternQuery(const std::string& pattern) override;
void
BuildWithFieldData(const std::vector<FieldDataPtr>& datas) override;
@@ -313,6 +317,9 @@ class InvertedIndexTantivy : public ScalarIndex<T> {
const Config& config) override;
protected:
const TargetBitmap
PatternQuery(const std::string& pattern) override;
void
finish();
internal/core/src/index/InvertedIndexTest.cpp
+21 -2
View File
@@ -76,6 +76,25 @@ using namespace milvus::segcore;
namespace milvus::test {
TEST(InvertedIndex, PatternMatchPlannerPolicy) {
index::InvertedIndexTantivy<std::string> index;
EXPECT_TRUE(index.SupportPatternMatch());
EXPECT_TRUE(index.ShouldUseOp(proto::plan::OpType::PrefixMatch));
EXPECT_FALSE(index.ShouldUseOp(proto::plan::OpType::RegexMatch));
EXPECT_TRUE(index.ShouldUseOp(proto::plan::OpType::Equal));
EXPECT_TRUE(index.ShouldUseOp(proto::plan::OpType::Match));
EXPECT_FALSE(index.ShouldUseOp(proto::plan::OpType::InnerMatch));
EXPECT_FALSE(index.ShouldUseOp(proto::plan::OpType::PostfixMatch));
index::InvertedIndexTantivy<int64_t> int_index;
EXPECT_FALSE(int_index.SupportPatternMatch());
EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::Match));
EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::PrefixMatch));
EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::RegexMatch));
EXPECT_TRUE(int_index.ShouldUseOp(proto::plan::OpType::Equal));
}
struct ChunkManagerWrapper {
ChunkManagerWrapper(storage::ChunkManagerPtr cm) : cm_(cm) {
}
@@ -889,9 +908,9 @@ test_string() {
}
{
ASSERT_TRUE(real_index->SupportPatternQuery());
auto prefix = data[0];
auto bitset = real_index->PatternQuery(prefix + "%");
auto bitset =
real_index->PatternMatch(prefix + "%", proto::plan::Match);
ASSERT_EQ(cnt, bitset.size());
size_t start = 0;
if (has_lack_binlog_row_) {
internal/core/src/index/JsonFlatIndex.h
+2 -1
View File
@@ -156,6 +156,7 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy<T> {
return bitset;
}
protected:
const TargetBitmap
PatternQuery(const std::string& pattern) override {
tracer::AutoSpan span("JsonFlatIndexQueryExecutor::PatternQuery",
@@ -239,4 +240,4 @@ JsonFlatIndexQueryExecutor<T>::JsonFlatIndexQueryExecutor(
this->wrapper_ = json_flat_index.wrapper_;
this->null_offset_ = json_flat_index.null_offset_;
}
} // namespace milvus::index
} // namespace milvus::index
internal/core/src/index/JsonFlatIndexTest.cpp
+3 -3
View File
@@ -295,21 +295,21 @@ TEST_F(JsonFlatIndexTest, TestPrefixMatchQuery) {
ASSERT_FALSE(result[2]); // Charlie doesn't start with A
}
TEST_F(JsonFlatIndexTest, TestPatternQuery) {
TEST_F(JsonFlatIndexTest, TestLikePatternMatch) {
auto json_flat_index =
dynamic_cast<index::JsonFlatIndex*>(json_index_.get());
ASSERT_NE(json_flat_index, nullptr);
std::string json_path = "/profile/name/first";
auto executor = json_flat_index->create_executor<std::string>(json_path);
auto result = executor->PatternQuery("A%ice");
auto result = executor->PatternMatch("A%ice", proto::plan::Match);
ASSERT_EQ(result.size(), json_data_.size());
ASSERT_TRUE(result[0]); // Alice matches A%ice
ASSERT_FALSE(result[1]); // Bob doesn't match A%ice
ASSERT_FALSE(result[2]); // Charlie doesn't match A%ice
// Test another LIKE pattern
auto result2 = executor->PatternQuery("B_b");
auto result2 = executor->PatternMatch("B_b", proto::plan::Match);
ASSERT_EQ(result2.size(), json_data_.size());
ASSERT_FALSE(result2[0]); // Alice doesn't match B_b
ASSERT_TRUE(result2[1]); // Bob matches B_b
internal/core/src/index/ScalarIndex.h
+25 -25
View File
@@ -193,32 +193,23 @@ class ScalarIndex : public IndexBase {
virtual int64_t
Size() = 0;
// Whether this index can execute a LIKE-pattern query internally.
// When true, the framework calls PatternQuery() directly instead of
// falling back to brute-force scan.
// NOTE: PatternQuery() always accepts a raw SQL LIKE pattern (e.g. "%hello%").
// Each index implementation is responsible for converting it to whatever
// internal format it needs (e.g., regex for tantivy).
// Planner policy for scalar-index execution. Most scalar ops can use the
// index directly. Pattern ops need extra capability checks:
// - PatternMatch is the public index capability for all pattern ops.
// - PatternQuery is an implementation detail used inside PatternMatch.
virtual bool
SupportPatternQuery() const {
return false;
}
// Whether the framework should *attempt* to use PatternQuery() for
// Match operations. Inverted indexes return false here because they
// handle Match via PatternMatch() dispatch instead.
virtual bool
TryUsePatternQuery() const {
return true;
}
// Execute a LIKE-pattern query on the index.
// @param pattern: a raw SQL LIKE pattern (e.g. "%hello%", "abc_def"),
// NOT a regex. Implementations must convert internally if needed
// (e.g., tantivy converts to regex via PatternMatchTranslator).
virtual const TargetBitmap
PatternQuery(const std::string& pattern) {
ThrowInfo(Unsupported, "pattern query is not supported");
ShouldUseOp(proto::plan::OpType op) const {
switch (op) {
case proto::plan::OpType::Match:
case proto::plan::OpType::PrefixMatch:
case proto::plan::OpType::PostfixMatch:
case proto::plan::OpType::InnerMatch:
case proto::plan::OpType::RegexMatch:
return std::is_same_v<T, std::string> &&
(SupportPatternMatch() || HasRawData());
default:
return true;
}
}
virtual void
@@ -257,6 +248,15 @@ class ScalarIndex : public IndexBase {
}
protected:
// Execute a LIKE-pattern query inside PatternMatch implementations.
// @param pattern: a raw SQL LIKE pattern (e.g. "%hello%", "abc_def"),
// NOT a regex. Implementations must convert internally if needed
// (e.g., tantivy converts to regex via PatternMatchTranslator).
virtual const TargetBitmap
PatternQuery(const std::string& pattern) {
ThrowInfo(Unsupported, "pattern query is not supported");
}
// File manager for V3 upload/load operations
storage::MemFileManagerImplPtr file_manager_;
internal/core/src/index/ScalarIndexTest.cpp
+26
View File
@@ -149,6 +149,32 @@ TYPED_TEST_P(TypedScalarIndexTest, HasRawData) {
}
}
TEST(ScalarIndexPlannerPolicy, PatternOpsRequireString) {
milvus::index::BitmapIndex<int64_t> int_index;
EXPECT_FALSE(int_index.ShouldUseOp(milvus::proto::plan::OpType::Match));
EXPECT_FALSE(
int_index.ShouldUseOp(milvus::proto::plan::OpType::PrefixMatch));
EXPECT_FALSE(
int_index.ShouldUseOp(milvus::proto::plan::OpType::PostfixMatch));
EXPECT_FALSE(
int_index.ShouldUseOp(milvus::proto::plan::OpType::InnerMatch));
EXPECT_FALSE(
int_index.ShouldUseOp(milvus::proto::plan::OpType::RegexMatch));
EXPECT_TRUE(int_index.ShouldUseOp(milvus::proto::plan::OpType::Equal));
milvus::index::BitmapIndex<std::string> string_index;
EXPECT_TRUE(string_index.ShouldUseOp(milvus::proto::plan::OpType::Match));
EXPECT_TRUE(
string_index.ShouldUseOp(milvus::proto::plan::OpType::PrefixMatch));
EXPECT_TRUE(
string_index.ShouldUseOp(milvus::proto::plan::OpType::PostfixMatch));
EXPECT_TRUE(
string_index.ShouldUseOp(milvus::proto::plan::OpType::InnerMatch));
EXPECT_TRUE(
string_index.ShouldUseOp(milvus::proto::plan::OpType::RegexMatch));
EXPECT_TRUE(string_index.ShouldUseOp(milvus::proto::plan::OpType::Equal));
}
TYPED_TEST_P(TypedScalarIndexTest, In) {
using T = TypeParam;
auto dtype = milvus::GetDType<T>();