From ccd649b2fc2d19dfcaff313a8e6a4004e7b1bc2f Mon Sep 17 00:00:00 2001 From: aoiasd <45024769+aoiasd@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:08:16 +0800 Subject: [PATCH] enhance: refine inverted pattern match planner policy (#49922) issue: #50005 ## Summary Separate pattern match capability from planner policy so inverted indexes keep regex filters on the index path while suffix/contains LIKE can fall back to raw-data scan when appropriate. Signed-off-by: aoiasd --- internal/core/src/common/RegexQueryTest.cpp | 5 -- internal/core/src/exec/expression/Expr.h | 23 ++---- internal/core/src/index/BitmapIndex.h | 82 +++++++++---------- internal/core/src/index/BitmapIndexTest.cpp | 53 +++++++++++- internal/core/src/index/HybridScalarIndex.h | 22 ++--- .../core/src/index/HybridScalarIndexTest.cpp | 16 ++++ .../core/src/index/InvertedIndexTantivy.h | 29 ++++--- internal/core/src/index/InvertedIndexTest.cpp | 23 +++++- internal/core/src/index/JsonFlatIndex.h | 3 +- internal/core/src/index/JsonFlatIndexTest.cpp | 6 +- internal/core/src/index/ScalarIndex.h | 50 +++++------ internal/core/src/index/ScalarIndexTest.cpp | 26 ++++++ 12 files changed, 212 insertions(+), 126 deletions(-) diff --git a/internal/core/src/common/RegexQueryTest.cpp b/internal/core/src/common/RegexQueryTest.cpp index 5129627dd0..43ca0e02ba 100644 --- a/internal/core/src/common/RegexQueryTest.cpp +++ b/internal/core/src/common/RegexQueryTest.cpp @@ -212,11 +212,6 @@ struct MockStringIndex : index::StringIndexMarisa { HasRawData() const override { return true; } - - bool - SupportPatternQuery() const override { - return false; - } }; class SealedSegmentRegexQueryTest : public ::testing::Test { diff --git a/internal/core/src/exec/expression/Expr.h b/internal/core/src/exec/expression/Expr.h index 4bfe067bcc..af185f6b97 100644 --- a/internal/core/src/exec/expression/Expr.h +++ b/internal/core/src/exec/expression/Expr.h @@ -2049,23 +2049,12 @@ class SegmentExpr : public Expr { } using Index = index::ScalarIndex; - if (op == OpType::Match || op == OpType::InnerMatch || - op == OpType::PostfixMatch || op == OpType::RegexMatch) { - AssertInfo(num_index_chunk_ == 1, - "scalar index should have exactly 1 chunk, got {}", - num_index_chunk_); - auto scalar_index = - dynamic_cast(pinned_index_[0].get()); - auto* index_ptr = const_cast(scalar_index); - // 1. index supports PatternMatch (iterates unique values) — preferred; - // 2. index supports PatternQuery via TryUsePatternQuery; - // 3. index has raw data for Reverse_Lookup fallback. - return index_ptr->SupportPatternMatch() || - (index_ptr->TryUsePatternQuery() && - index_ptr->SupportPatternQuery()) || - index_ptr->HasRawData(); - } - return true; + AssertInfo(num_index_chunk_ == 1, + "scalar index should have exactly 1 chunk, got {}", + num_index_chunk_); + auto scalar_index = dynamic_cast(pinned_index_[0].get()); + AssertInfo(scalar_index != nullptr, "invalid scalar index type"); + return scalar_index->ShouldUseOp(op); } template diff --git a/internal/core/src/index/BitmapIndex.h b/internal/core/src/index/BitmapIndex.h index cbcc07fa44..6cb776c542 100644 --- a/internal/core/src/index/BitmapIndex.h +++ b/internal/core/src/index/BitmapIndex.h @@ -207,49 +207,6 @@ class BitmapIndex : public ScalarIndex { return std::is_same_v; } - bool - SupportPatternQuery() const override { - return std::is_same_v; - } - - const TargetBitmap - PatternQuery(const std::string& pattern) override { - if constexpr (!std::is_same_v) { - ThrowInfo(ErrorCode::OpTypeInvalid, - "pattern query only supported for string type"); - return TargetBitmap{}; - } else { - AssertInfo(is_built_, "index has not been built"); - - LikePatternMatcher matcher(pattern); - TargetBitmap res(total_num_rows_, false); - if (is_mmap_) { - for (const auto& [key, bitmap] : bitmap_info_map_) { - if (matcher(key)) { - for (const auto& v : bitmap) { - res.set(v); - } - } - } - } else if (build_mode_ == BitmapIndexBuildMode::ROARING) { - for (const auto& [key, bitmap] : data_) { - if (matcher(key)) { - for (const auto& v : bitmap) { - res.set(v); - } - } - } - } else { - for (const auto& [key, bitset] : bitsets_) { - if (matcher(key)) { - res |= bitset; - } - } - } - return res; - } - } - const TargetBitmap PatternMatch(const std::string& pattern, proto::plan::OpType op) override { switch (op) { @@ -306,6 +263,45 @@ class BitmapIndex : public ScalarIndex { } } + protected: + const TargetBitmap + PatternQuery(const std::string& pattern) override { + if constexpr (!std::is_same_v) { + ThrowInfo(ErrorCode::OpTypeInvalid, + "pattern query only supported for string type"); + return TargetBitmap{}; + } + + AssertInfo(is_built_, "index has not been built"); + + LikePatternMatcher matcher(pattern); + TargetBitmap res(total_num_rows_, false); + if (is_mmap_) { + for (const auto& [key, bitmap] : bitmap_info_map_) { + if (matcher(key)) { + for (const auto& v : bitmap) { + res.set(v); + } + } + } + } else if (build_mode_ == BitmapIndexBuildMode::ROARING) { + for (const auto& [key, bitmap] : data_) { + if (matcher(key)) { + for (const auto& v : bitmap) { + res.set(v); + } + } + } + } else { + for (const auto& [key, bitset] : bitsets_) { + if (matcher(key)) { + res |= bitset; + } + } + } + return res; + } + public: int64_t Cardinality() { diff --git a/internal/core/src/index/BitmapIndexTest.cpp b/internal/core/src/index/BitmapIndexTest.cpp index 1e2d9b5484..1f464cf25a 100644 --- a/internal/core/src/index/BitmapIndexTest.cpp +++ b/internal/core/src/index/BitmapIndexTest.cpp @@ -524,6 +524,38 @@ class BitmapIndexTest : public testing::Test { } } + void + TestPatternMatchFunc() { + if constexpr (std::is_same_v) { + auto index_ptr = dynamic_cast*>(index_.get()); + auto like_bitset = + index_ptr->PatternMatch("1%", proto::plan::OpType::Match); + auto regex_bitset = index_ptr->PatternMatch( + "^1.*", proto::plan::OpType::RegexMatch); + ASSERT_EQ(like_bitset.size(), index_ptr->Count()); + ASSERT_EQ(regex_bitset.size(), index_ptr->Count()); + + size_t start = 0; + if (has_lack_binlog_row_) { + for (int i = 0; i < lack_binlog_row_; i++) { + ASSERT_FALSE(like_bitset[i]); + ASSERT_FALSE(regex_bitset[i]); + } + start += lack_binlog_row_; + } + + for (size_t i = start; i < like_bitset.size(); i++) { + auto data_offset = i - start; + auto expected = data_[data_offset].rfind("1", 0) == 0; + if (nullable_ && !valid_data_[data_offset]) { + expected = false; + } + ASSERT_EQ(like_bitset[i], expected); + ASSERT_EQ(regex_bitset[i], expected); + } + } + } + public: IndexBasePtr index_; DataType type_; @@ -569,6 +601,10 @@ TYPED_TEST_P(BitmapIndexTest, IsNotNullFuncTest) { this->TestIsNotNullFunc(); } +TYPED_TEST_P(BitmapIndexTest, PatternMatchFuncTest) { + this->TestPatternMatchFunc(); +} + using BitmapType = testing::Types; @@ -578,7 +614,8 @@ REGISTER_TYPED_TEST_SUITE_P(BitmapIndexTest, NotINFuncTest, CompareValFuncTest, IsNullFuncTest, - IsNotNullFuncTest); + IsNotNullFuncTest, + PatternMatchFuncTest); INSTANTIATE_TYPED_TEST_SUITE_P(BitmapE2ECheck, BitmapIndexTest, BitmapType); @@ -629,6 +666,10 @@ TYPED_TEST_P(BitmapIndexTestV2, IsNotNullFuncTest) { this->TestIsNotNullFunc(); } +TYPED_TEST_P(BitmapIndexTestV2, PatternMatchFuncTest) { + this->TestPatternMatchFunc(); +} + using BitmapType = testing::Types; @@ -639,7 +680,8 @@ REGISTER_TYPED_TEST_SUITE_P(BitmapIndexTestV2, CompareValFuncTest, TestRangeCompareFuncTest, IsNullFuncTest, - IsNotNullFuncTest); + IsNotNullFuncTest, + PatternMatchFuncTest); INSTANTIATE_TYPED_TEST_SUITE_P(BitmapIndexE2ECheck_HighCardinality, BitmapIndexTestV2, @@ -693,6 +735,10 @@ TYPED_TEST_P(BitmapIndexTestV3, IsNotNullFuncTest) { this->TestIsNotNullFunc(); } +TYPED_TEST_P(BitmapIndexTestV3, PatternMatchFuncTest) { + this->TestPatternMatchFunc(); +} + using BitmapType = testing::Types; @@ -703,7 +749,8 @@ REGISTER_TYPED_TEST_SUITE_P(BitmapIndexTestV3, CompareValFuncTest, TestRangeCompareFuncTest, IsNullFuncTest, - IsNotNullFuncTest); + IsNotNullFuncTest, + PatternMatchFuncTest); INSTANTIATE_TYPED_TEST_SUITE_P(BitmapIndexE2ECheck_Mmap, BitmapIndexTestV3, diff --git a/internal/core/src/index/HybridScalarIndex.h b/internal/core/src/index/HybridScalarIndex.h index 6fee75fcbc..642cc74228 100644 --- a/internal/core/src/index/HybridScalarIndex.h +++ b/internal/core/src/index/HybridScalarIndex.h @@ -110,6 +110,11 @@ class HybridScalarIndex : public ScalarIndex { return internal_index_->Query(dataset); } + bool + ShouldUseOp(proto::plan::OpType op) const override { + return internal_index_->ShouldUseOp(op); + } + bool SupportPatternMatch() const override { return internal_index_->SupportPatternMatch(); @@ -120,21 +125,6 @@ class HybridScalarIndex : public ScalarIndex { return internal_index_->PatternMatch(pattern, op); } - bool - TryUsePatternQuery() const override { - return internal_index_->TryUsePatternQuery(); - } - - bool - SupportPatternQuery() const override { - return internal_index_->SupportPatternQuery(); - } - - const TargetBitmap - PatternQuery(const std::string& pattern) override { - return internal_index_->PatternQuery(pattern); - } - const TargetBitmap Range(const T& value, OpType op) override { return internal_index_->Range(value, op); @@ -239,4 +229,4 @@ class HybridScalarIndex : public ScalarIndex { }; } // namespace index -} // namespace milvus \ No newline at end of file +} // namespace milvus diff --git a/internal/core/src/index/HybridScalarIndexTest.cpp b/internal/core/src/index/HybridScalarIndexTest.cpp index e31735cf4e..58c0673519 100644 --- a/internal/core/src/index/HybridScalarIndexTest.cpp +++ b/internal/core/src/index/HybridScalarIndexTest.cpp @@ -56,6 +56,22 @@ using namespace milvus::indexbuilder; using namespace milvus; using namespace milvus::index; +TEST(HybridScalarIndexPlannerPolicy, ShouldUseOpDelegatesToInternalIndex) { + HybridScalarIndex int_index(7); + std::vector int_data{1, 2, 3, 4}; + int_index.Build(int_data.size(), int_data.data()); + EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::Match)); + EXPECT_TRUE(int_index.ShouldUseOp(proto::plan::OpType::Equal)); + + HybridScalarIndex string_index(7); + std::vector string_data{"alpha", "beta", "alphabet"}; + string_index.Build(string_data.size(), string_data.data()); + EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::Match)); + EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::PrefixMatch)); + EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::RegexMatch)); + EXPECT_TRUE(string_index.ShouldUseOp(proto::plan::OpType::Equal)); +} + template static std::vector GenerateData(const size_t size, const size_t cardinality) { diff --git a/internal/core/src/index/InvertedIndexTantivy.h b/internal/core/src/index/InvertedIndexTantivy.h index b6ca411466..c94292904c 100644 --- a/internal/core/src/index/InvertedIndexTantivy.h +++ b/internal/core/src/index/InvertedIndexTantivy.h @@ -284,19 +284,23 @@ class InvertedIndexTantivy : public ScalarIndex { } bool - SupportPatternQuery() const override { - return std::is_same_v; + ShouldUseOp(proto::plan::OpType op) const override { + // Suffix/contains LIKE and regex can be executed correctly over indexed + // terms, but for short varchar values they are often slower than raw + // scan. Keep only prefix/LIKE Match on the inverted-index path. + switch (op) { + case proto::plan::OpType::Match: + case proto::plan::OpType::PrefixMatch: + return SupportPatternMatch() || HasRawData(); + case proto::plan::OpType::RegexMatch: + case proto::plan::OpType::PostfixMatch: + case proto::plan::OpType::InnerMatch: + return false; + default: + return true; + } } - bool - TryUsePatternQuery() const override { - // for inverted index, not use pattern query to implement match - return false; - } - - const TargetBitmap - PatternQuery(const std::string& pattern) override; - void BuildWithFieldData(const std::vector& datas) override; @@ -313,6 +317,9 @@ class InvertedIndexTantivy : public ScalarIndex { const Config& config) override; protected: + const TargetBitmap + PatternQuery(const std::string& pattern) override; + void finish(); diff --git a/internal/core/src/index/InvertedIndexTest.cpp b/internal/core/src/index/InvertedIndexTest.cpp index 74d1a850e1..921b6c0416 100644 --- a/internal/core/src/index/InvertedIndexTest.cpp +++ b/internal/core/src/index/InvertedIndexTest.cpp @@ -76,6 +76,25 @@ using namespace milvus::segcore; namespace milvus::test { +TEST(InvertedIndex, PatternMatchPlannerPolicy) { + index::InvertedIndexTantivy index; + + EXPECT_TRUE(index.SupportPatternMatch()); + EXPECT_TRUE(index.ShouldUseOp(proto::plan::OpType::PrefixMatch)); + EXPECT_FALSE(index.ShouldUseOp(proto::plan::OpType::RegexMatch)); + EXPECT_TRUE(index.ShouldUseOp(proto::plan::OpType::Equal)); + EXPECT_TRUE(index.ShouldUseOp(proto::plan::OpType::Match)); + EXPECT_FALSE(index.ShouldUseOp(proto::plan::OpType::InnerMatch)); + EXPECT_FALSE(index.ShouldUseOp(proto::plan::OpType::PostfixMatch)); + + index::InvertedIndexTantivy int_index; + EXPECT_FALSE(int_index.SupportPatternMatch()); + EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::Match)); + EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::PrefixMatch)); + EXPECT_FALSE(int_index.ShouldUseOp(proto::plan::OpType::RegexMatch)); + EXPECT_TRUE(int_index.ShouldUseOp(proto::plan::OpType::Equal)); +} + struct ChunkManagerWrapper { ChunkManagerWrapper(storage::ChunkManagerPtr cm) : cm_(cm) { } @@ -889,9 +908,9 @@ test_string() { } { - ASSERT_TRUE(real_index->SupportPatternQuery()); auto prefix = data[0]; - auto bitset = real_index->PatternQuery(prefix + "%"); + auto bitset = + real_index->PatternMatch(prefix + "%", proto::plan::Match); ASSERT_EQ(cnt, bitset.size()); size_t start = 0; if (has_lack_binlog_row_) { diff --git a/internal/core/src/index/JsonFlatIndex.h b/internal/core/src/index/JsonFlatIndex.h index aa4861b590..d9d4660401 100644 --- a/internal/core/src/index/JsonFlatIndex.h +++ b/internal/core/src/index/JsonFlatIndex.h @@ -156,6 +156,7 @@ class JsonFlatIndexQueryExecutor : public InvertedIndexTantivy { return bitset; } + protected: const TargetBitmap PatternQuery(const std::string& pattern) override { tracer::AutoSpan span("JsonFlatIndexQueryExecutor::PatternQuery", @@ -239,4 +240,4 @@ JsonFlatIndexQueryExecutor::JsonFlatIndexQueryExecutor( this->wrapper_ = json_flat_index.wrapper_; this->null_offset_ = json_flat_index.null_offset_; } -} // namespace milvus::index \ No newline at end of file +} // namespace milvus::index diff --git a/internal/core/src/index/JsonFlatIndexTest.cpp b/internal/core/src/index/JsonFlatIndexTest.cpp index 807ba9450a..935fea54cf 100644 --- a/internal/core/src/index/JsonFlatIndexTest.cpp +++ b/internal/core/src/index/JsonFlatIndexTest.cpp @@ -295,21 +295,21 @@ TEST_F(JsonFlatIndexTest, TestPrefixMatchQuery) { ASSERT_FALSE(result[2]); // Charlie doesn't start with A } -TEST_F(JsonFlatIndexTest, TestPatternQuery) { +TEST_F(JsonFlatIndexTest, TestLikePatternMatch) { auto json_flat_index = dynamic_cast(json_index_.get()); ASSERT_NE(json_flat_index, nullptr); std::string json_path = "/profile/name/first"; auto executor = json_flat_index->create_executor(json_path); - auto result = executor->PatternQuery("A%ice"); + auto result = executor->PatternMatch("A%ice", proto::plan::Match); ASSERT_EQ(result.size(), json_data_.size()); ASSERT_TRUE(result[0]); // Alice matches A%ice ASSERT_FALSE(result[1]); // Bob doesn't match A%ice ASSERT_FALSE(result[2]); // Charlie doesn't match A%ice // Test another LIKE pattern - auto result2 = executor->PatternQuery("B_b"); + auto result2 = executor->PatternMatch("B_b", proto::plan::Match); ASSERT_EQ(result2.size(), json_data_.size()); ASSERT_FALSE(result2[0]); // Alice doesn't match B_b ASSERT_TRUE(result2[1]); // Bob matches B_b diff --git a/internal/core/src/index/ScalarIndex.h b/internal/core/src/index/ScalarIndex.h index 578a5a7b19..e28a0ef649 100644 --- a/internal/core/src/index/ScalarIndex.h +++ b/internal/core/src/index/ScalarIndex.h @@ -193,32 +193,23 @@ class ScalarIndex : public IndexBase { virtual int64_t Size() = 0; - // Whether this index can execute a LIKE-pattern query internally. - // When true, the framework calls PatternQuery() directly instead of - // falling back to brute-force scan. - // NOTE: PatternQuery() always accepts a raw SQL LIKE pattern (e.g. "%hello%"). - // Each index implementation is responsible for converting it to whatever - // internal format it needs (e.g., regex for tantivy). + // Planner policy for scalar-index execution. Most scalar ops can use the + // index directly. Pattern ops need extra capability checks: + // - PatternMatch is the public index capability for all pattern ops. + // - PatternQuery is an implementation detail used inside PatternMatch. virtual bool - SupportPatternQuery() const { - return false; - } - - // Whether the framework should *attempt* to use PatternQuery() for - // Match operations. Inverted indexes return false here because they - // handle Match via PatternMatch() dispatch instead. - virtual bool - TryUsePatternQuery() const { - return true; - } - - // Execute a LIKE-pattern query on the index. - // @param pattern: a raw SQL LIKE pattern (e.g. "%hello%", "abc_def"), - // NOT a regex. Implementations must convert internally if needed - // (e.g., tantivy converts to regex via PatternMatchTranslator). - virtual const TargetBitmap - PatternQuery(const std::string& pattern) { - ThrowInfo(Unsupported, "pattern query is not supported"); + ShouldUseOp(proto::plan::OpType op) const { + switch (op) { + case proto::plan::OpType::Match: + case proto::plan::OpType::PrefixMatch: + case proto::plan::OpType::PostfixMatch: + case proto::plan::OpType::InnerMatch: + case proto::plan::OpType::RegexMatch: + return std::is_same_v && + (SupportPatternMatch() || HasRawData()); + default: + return true; + } } virtual void @@ -257,6 +248,15 @@ class ScalarIndex : public IndexBase { } protected: + // Execute a LIKE-pattern query inside PatternMatch implementations. + // @param pattern: a raw SQL LIKE pattern (e.g. "%hello%", "abc_def"), + // NOT a regex. Implementations must convert internally if needed + // (e.g., tantivy converts to regex via PatternMatchTranslator). + virtual const TargetBitmap + PatternQuery(const std::string& pattern) { + ThrowInfo(Unsupported, "pattern query is not supported"); + } + // File manager for V3 upload/load operations storage::MemFileManagerImplPtr file_manager_; diff --git a/internal/core/src/index/ScalarIndexTest.cpp b/internal/core/src/index/ScalarIndexTest.cpp index e0578d3aa7..0e6644e750 100644 --- a/internal/core/src/index/ScalarIndexTest.cpp +++ b/internal/core/src/index/ScalarIndexTest.cpp @@ -149,6 +149,32 @@ TYPED_TEST_P(TypedScalarIndexTest, HasRawData) { } } +TEST(ScalarIndexPlannerPolicy, PatternOpsRequireString) { + milvus::index::BitmapIndex int_index; + EXPECT_FALSE(int_index.ShouldUseOp(milvus::proto::plan::OpType::Match)); + EXPECT_FALSE( + int_index.ShouldUseOp(milvus::proto::plan::OpType::PrefixMatch)); + EXPECT_FALSE( + int_index.ShouldUseOp(milvus::proto::plan::OpType::PostfixMatch)); + EXPECT_FALSE( + int_index.ShouldUseOp(milvus::proto::plan::OpType::InnerMatch)); + EXPECT_FALSE( + int_index.ShouldUseOp(milvus::proto::plan::OpType::RegexMatch)); + EXPECT_TRUE(int_index.ShouldUseOp(milvus::proto::plan::OpType::Equal)); + + milvus::index::BitmapIndex string_index; + EXPECT_TRUE(string_index.ShouldUseOp(milvus::proto::plan::OpType::Match)); + EXPECT_TRUE( + string_index.ShouldUseOp(milvus::proto::plan::OpType::PrefixMatch)); + EXPECT_TRUE( + string_index.ShouldUseOp(milvus::proto::plan::OpType::PostfixMatch)); + EXPECT_TRUE( + string_index.ShouldUseOp(milvus::proto::plan::OpType::InnerMatch)); + EXPECT_TRUE( + string_index.ShouldUseOp(milvus::proto::plan::OpType::RegexMatch)); + EXPECT_TRUE(string_index.ShouldUseOp(milvus::proto::plan::OpType::Equal)); +} + TYPED_TEST_P(TypedScalarIndexTest, In) { using T = TypeParam; auto dtype = milvus::GetDType();