mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
fix: implement PatternMatch for StringIndexMarisa to fix LIKE prefix performance regression (#48686)
StringIndexMarisa (Trie index) was missing SupportPatternMatch() and PatternMatch() overrides, causing the expression evaluation framework to bypass the trie's efficient predictive_search-based PrefixMatch and fall back to O(n) brute-force scan via Reverse_Lookup for every row. This regression was introduced when the expression framework was refactored to use UnaryIndexFunc/UnaryIndexFuncForMatch instead of the old StringIndex::Query(dataset) path. Other index types (StringIndexSort, InvertedIndexTantivy, BitmapIndex) were updated to implement these interfaces, but StringIndexMarisa was missed. issue: https://github.com/milvus-io/milvus/issues/48685 Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
@@ -36,6 +36,7 @@
|
||||
#include "common/File.h"
|
||||
#include "common/Slice.h"
|
||||
#include "common/Tracer.h"
|
||||
#include "common/RegexQuery.h"
|
||||
#include "common/Types.h"
|
||||
#include "common/Utils.h"
|
||||
#include "fmt/core.h"
|
||||
@@ -594,6 +595,60 @@ StringIndexMarisa::PrefixMatch(std::string_view prefix) {
|
||||
return bitset;
|
||||
}
|
||||
|
||||
const TargetBitmap
|
||||
StringIndexMarisa::PatternMatch(const std::string& pattern,
|
||||
proto::plan::OpType op) {
|
||||
if (op == proto::plan::OpType::PrefixMatch) {
|
||||
return PrefixMatch(pattern);
|
||||
}
|
||||
|
||||
if (op != proto::plan::OpType::Match &&
|
||||
op != proto::plan::OpType::PostfixMatch &&
|
||||
op != proto::plan::OpType::InnerMatch) {
|
||||
ThrowInfo(Unsupported,
|
||||
"StringIndexMarisa::PatternMatch only supports Match, "
|
||||
"PrefixMatch, PostfixMatch, InnerMatch, got op: {}",
|
||||
static_cast<int>(op));
|
||||
}
|
||||
|
||||
// For Match/PostfixMatch/InnerMatch, iterate over unique trie keys
|
||||
// instead of all rows to avoid redundant matching on duplicate values.
|
||||
TargetBitmap bitset(str_ids_.size());
|
||||
|
||||
auto match_fn = [&](const std::string& val) -> bool {
|
||||
switch (op) {
|
||||
case proto::plan::OpType::PostfixMatch:
|
||||
return PostfixMatch(val, pattern);
|
||||
case proto::plan::OpType::InnerMatch:
|
||||
return InnerMatch(val, pattern);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
if (op == proto::plan::OpType::Match) {
|
||||
LikePatternMatcher matcher(pattern);
|
||||
for (const auto& [str_id, offsets] : str_ids_to_offsets_) {
|
||||
auto val = Reverse_Lookup(offsets[0]);
|
||||
if (val.has_value() && matcher(val.value())) {
|
||||
for (auto offset : offsets) {
|
||||
bitset[offset] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const auto& [str_id, offsets] : str_ids_to_offsets_) {
|
||||
auto val = Reverse_Lookup(offsets[0]);
|
||||
if (val.has_value() && match_fn(val.value())) {
|
||||
for (auto offset : offsets) {
|
||||
bitset[offset] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return bitset;
|
||||
}
|
||||
|
||||
void
|
||||
StringIndexMarisa::fill_str_ids(size_t n,
|
||||
const std::string* values,
|
||||
|
||||
@@ -92,6 +92,14 @@ class StringIndexMarisa : public StringIndex {
|
||||
const TargetBitmap
|
||||
PrefixMatch(const std::string_view prefix) override;
|
||||
|
||||
bool
|
||||
SupportPatternMatch() const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
const TargetBitmap
|
||||
PatternMatch(const std::string& pattern, proto::plan::OpType op) override;
|
||||
|
||||
std::optional<std::string>
|
||||
Reverse_Lookup(size_t offset) const override;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user