enhance: [3.0] move JSON indexes into sealed segment runtime state (#51323) (#51413)

Cherry pick from master
pr: #51323
Related to #51068

- manage JSON index load, replace, and drop through reopen COW snapshots
- remove LoadScope_Index and the legacy QueryCoord-to-segcore update
path
- drop JSON indexes by field and nested path to preserve sibling indexes
- regenerate query proto and QueryNode mocks

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia
2026-07-16 17:44:38 +08:00
committed by GitHub
parent ca8304b376
commit 4385b8a2c1
28 changed files with 1327 additions and 1475 deletions
@@ -52,6 +52,30 @@ This design does **not** require the following:
The goal is **one unified direction with phased delivery**, not a one-shot rewrite of the entire sealed segment implementation.
### 1.4 Rolling Upgrade Compatibility
The index-update migration assumes a coordinator baseline whose IndexChecker
schedules segment index changes through `LoadScope_Reopen`. The historical
`LoadScope_Index` wire value (`2`) is retired and remains reserved so it cannot
be reused for another meaning.
This design does not support a mixed-version deployment in which an older
QueryCoord still emits `LoadScope_Index` requests to a newer QueryNode after the
legacy index-update handler has been removed. A QueryNode that receives wire
value `2` rejects it as an internal protocol mismatch with an explicit upgrade
message. The deployment must upgrade QueryCoord to a version that emits
`LoadScope_Reopen` before routing index-update requests to these QueryNodes.
The retired value must not fall through to the full segment load path: an index
update is not a full load and must not trigger full-load distribution, bloom
filter, or delete-stream side effects. It must also not be blindly remapped to
`LoadScope_Reopen`, because a legacy request may omit newer
`SegmentLoadInfo` fields; publishing that request as the new load-info snapshot
could discard metadata already held by the newer QueryNode. Supporting that
version skew would require a dedicated compatibility adapter that merges and
normalizes the legacy request against the current published load info before
reopen, which is outside the scope of this design.
---
## 2. Core Problem
@@ -263,7 +263,7 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) {
auto cload_index_info =
static_cast<CLoadIndexInfo>(&load_index_info);
AppendIndexV2(trace, cload_index_info);
UpdateSealedSegmentIndex(segment.get(), cload_index_info);
segment->LoadIndex(load_index_info);
};
// Load pk field
@@ -284,7 +284,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
trace.traceFlags = 0;
auto cload_index_info = static_cast<CLoadIndexInfo>(&load_index_info);
AppendIndexV2(trace, cload_index_info);
UpdateSealedSegmentIndex(segment.get(), cload_index_info);
segment->LoadIndex(load_index_info);
auto unary_range_expr = test::GenUnaryRangeExpr(op_type, literal);
auto column_info = test::GenColumnInfo(
@@ -562,7 +562,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
trace.traceFlags = 0;
auto cload_index_info = static_cast<CLoadIndexInfo>(&load_index_info);
AppendIndexV2(trace, cload_index_info);
UpdateSealedSegmentIndex(segment.get(), cload_index_info);
segment->LoadIndex(load_index_info);
// Test: TermFilterExpr (IN operator)
{
@@ -290,32 +290,6 @@ cancel_and_clear_ngram_indexings(
ngram_indexings.clear();
}
template <typename JsonIndexT>
static void
cancel_and_erase_json_indices(std::vector<JsonIndexT>& json_indices,
FieldId field_id,
std::string_view nested_path) {
auto new_end = std::remove_if(
json_indices.begin(), json_indices.end(), [&](auto& index) {
auto matched =
index.field_id == field_id && index.nested_path == nested_path;
if (matched) {
cancel_warmup(index.index);
}
return matched;
});
json_indices.erase(new_end, json_indices.end());
}
template <typename JsonIndexT>
static void
cancel_and_clear_json_indices(std::vector<JsonIndexT>& json_indices) {
for (auto& index : json_indices) {
cancel_warmup(index.index);
}
json_indices.clear();
}
PinWrapper<const storagev2translator::TimestampIndexCell*>
ChunkedSegmentSealedImpl::PinTimestampIndex(
const std::shared_ptr<const RuntimeResourceState>& runtime,
@@ -370,6 +344,84 @@ ChunkedSegmentSealedImpl::PinPkIndex(milvus::OpContext* op_ctx) const {
return PinWrapper<const storagev2translator::PkIndexCell*>(ca, cell);
}
std::vector<PinWrapper<const index::IndexBase*>>
ChunkedSegmentSealedImpl::PinJsonIndex(milvus::OpContext* op_ctx,
FieldId field_id,
const std::string& path,
DataType data_type,
bool any_type,
bool is_array) const {
auto runtime = CaptureRuntimeResourceState();
int path_len_diff = std::numeric_limits<int>::max();
index::CacheIndexBasePtr best_match = nullptr;
std::string_view path_view = path;
for (const auto& index : runtime->json_indices) {
if (index.field_id != field_id) {
continue;
}
switch (index.cast_type.data_type()) {
case JsonCastType::DataType::JSON:
if (path_view.length() < index.nested_path.length()) {
continue;
}
if (path_view.substr(0, index.nested_path.length()) ==
index.nested_path) {
int current_len_diff =
path_view.length() - index.nested_path.length();
if (current_len_diff < path_len_diff) {
path_len_diff = current_len_diff;
best_match = index.index;
}
if (path_len_diff == 0) {
break;
}
}
break;
default:
if (index.nested_path != path) {
continue;
}
if (any_type || milvus::index::json::IsDataTypeSupported(
index.cast_type, data_type, is_array)) {
best_match = index.index;
}
break;
}
}
if (best_match == nullptr) {
return {};
}
auto ca = SemiInlineGet(best_match->PinCells(op_ctx, {0}));
auto pinned_index = ca->get_cell_of(0);
return {PinWrapper<const index::IndexBase*>(std::move(ca), pinned_index)};
}
std::string
ChunkedSegmentSealedImpl::GetJsonFlatIndexNestedPath(
FieldId field_id, std::string_view query_path) const {
auto runtime = CaptureRuntimeResourceState();
std::string best_path;
int path_len_diff = std::numeric_limits<int>::max();
for (const auto& index : runtime->json_indices) {
if (index.field_id != field_id ||
index.cast_type.data_type() != JsonCastType::DataType::JSON ||
query_path.length() < index.nested_path.length() ||
query_path.substr(0, index.nested_path.length()) !=
index.nested_path) {
continue;
}
int current_len_diff = query_path.length() - index.nested_path.length();
if (current_len_diff < path_len_diff) {
path_len_diff = current_len_diff;
best_path = index.nested_path;
}
if (path_len_diff == 0) {
break;
}
}
return best_path;
}
bool
ChunkedSegmentSealedImpl::Contain(const PkType& pk) const {
auto schema_snapshot = CaptureSchemaSnapshot();
@@ -593,8 +645,12 @@ ChunkedSegmentSealedImpl::LoadIndex(LoadIndexInfo& info,
LoadVecIndex(
info, schema_snapshot, is_replace, staged_state, committer);
} else {
LoadScalarIndex(
info, schema_snapshot, is_replace, runtime, staged_state);
LoadScalarIndex(info,
schema_snapshot,
is_replace,
runtime,
staged_state,
committer);
}
}
@@ -705,7 +761,8 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
const SchemaPtr& schema_snapshot,
bool is_replace,
RuntimeResourceState* runtime,
PublishedSegmentState* staged_state) {
PublishedSegmentState* staged_state,
StagedStateCommitter* committer) {
// NOTE: lock only when data is ready to avoid starvation
auto field_id = FieldId(info.field_id);
auto snapshot = CapturePublishedState();
@@ -716,6 +773,26 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
RuntimeResourceState* target_runtime = runtime;
std::shared_ptr<RuntimeResourceState> owned_runtime;
std::vector<index::CacheIndexBasePtr> retired_indexings;
auto retire_indexing = [&](index::CacheIndexBasePtr indexing) {
if (indexing == nullptr) {
return;
}
if (committer != nullptr) {
committer->RetireCacheIndexingLocked(std::move(indexing));
} else if (owned_runtime != nullptr) {
retired_indexings.push_back(std::move(indexing));
}
};
auto cancel_retired_indexings = [&] {
for (auto& indexing : retired_indexings) {
if (indexing != nullptr) {
indexing->CancelWarmup();
}
}
};
LOG_INFO("LoadScalarIndex, fieldID:{}. segmentID:{}, is_pk:{}",
info.field_id,
@@ -754,19 +831,19 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
if (field_meta.get_data_type() == DataType::JSON) {
auto path = info.index_params.at(JSON_PATH);
if (target_runtime == nullptr) {
owned_runtime = CloneMutableRuntimeResourceState();
target_runtime = owned_runtime.get();
}
for (auto& retired :
EraseJsonIndexesAtPath(*target_runtime, field_id, path)) {
retire_indexing(std::move(retired));
}
if (auto it = info.index_params.find(index::INDEX_TYPE);
it != info.index_params.end() &&
it->second == index::NGRAM_INDEX_TYPE) {
if (target_runtime == nullptr) {
owned_runtime = CloneMutableRuntimeResourceState();
target_runtime = owned_runtime.get();
}
auto& path_indexings = target_runtime->ngram_indexings[field_id];
if (auto path_it = path_indexings.find(path);
path_it != path_indexings.end()) {
cancel_warmup(path_it->second);
}
path_indexings[path] = std::move(info.cache_index);
target_runtime->ngram_indexings[field_id][path] =
std::move(info.cache_index);
if (staged_state != nullptr) {
clear_bit_if_present(
staged_state->published_binlog_index_ready_bitset,
@@ -781,6 +858,7 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
? ToConstRuntimeState(std::move(owned_runtime))
: FreezeRuntimeResourceState(*target_runtime);
PublishIndexReadyLocked(field_id, false, published_runtime);
cancel_retired_indexings();
}
return;
} else {
@@ -790,10 +868,21 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
index.index = std::move(info.cache_index);
index.cast_type =
JsonCastType::FromString(info.index_params.at(JSON_CAST_TYPE));
json_indices.withWLock([&](auto& json_indexings) {
cancel_and_erase_json_indices(json_indexings, field_id, path);
json_indexings.push_back(std::move(index));
});
target_runtime->json_indices.push_back(std::move(index));
if (staged_state != nullptr) {
SyncJsonNgramIndexState(
*staged_state, *target_runtime, field_id);
NormalizePublishedState(*staged_state);
} else if (owned_runtime != nullptr) {
auto published_runtime =
ToConstRuntimeState(std::move(owned_runtime));
MutatePublishedStateLocked([&](PublishedSegmentState& state) {
state.runtime = published_runtime;
SyncJsonNgramIndexState(
state, *published_runtime, field_id);
});
cancel_retired_indexings();
}
return;
}
}
@@ -904,6 +993,7 @@ ChunkedSegmentSealedImpl::CloneRuntimeResourceState(
state->ngram_indexings = current->ngram_indexings;
state->text_lob_paths = current->text_lob_paths;
state->text_indexes = current->text_indexes;
state->json_indices = current->json_indices;
state->json_stats = current->json_stats;
state->reader = current->reader;
state->timestamps = current->timestamps;
@@ -1013,6 +1103,87 @@ ChunkedSegmentSealedImpl::DropVectorIndexing(RuntimeResourceState& runtime,
}
}
std::vector<index::CacheIndexBasePtr>
ChunkedSegmentSealedImpl::EraseJsonIndexings(RuntimeResourceState& runtime,
FieldId field_id,
std::string_view nested_path) {
std::vector<index::CacheIndexBasePtr> retired;
auto new_end = std::remove_if(runtime.json_indices.begin(),
runtime.json_indices.end(),
[&](JsonIndex& index) {
if (index.field_id != field_id ||
index.nested_path != nested_path) {
return false;
}
retired.push_back(std::move(index.index));
return true;
});
runtime.json_indices.erase(new_end, runtime.json_indices.end());
return retired;
}
index::CacheIndexBasePtr
ChunkedSegmentSealedImpl::EraseJsonNgramIndexing(RuntimeResourceState& runtime,
FieldId field_id,
std::string_view nested_path) {
auto field_it = runtime.ngram_indexings.find(field_id);
if (field_it == runtime.ngram_indexings.end()) {
return nullptr;
}
auto& path_indexings = field_it->second;
auto path_it = path_indexings.find(std::string(nested_path));
if (path_it == path_indexings.end()) {
return nullptr;
}
auto retired = std::move(path_it->second);
path_indexings.erase(path_it);
if (path_indexings.empty()) {
runtime.ngram_indexings.erase(field_it);
}
return retired;
}
std::vector<index::CacheIndexBasePtr>
ChunkedSegmentSealedImpl::EraseJsonIndexesAtPath(RuntimeResourceState& runtime,
FieldId field_id,
std::string_view nested_path) {
auto retired = EraseJsonIndexings(runtime, field_id, nested_path);
if (auto ngram = EraseJsonNgramIndexing(runtime, field_id, nested_path);
ngram != nullptr) {
retired.push_back(std::move(ngram));
}
return retired;
}
bool
ChunkedSegmentSealedImpl::RuntimeJsonNgramIndexReady(
const RuntimeResourceState& runtime, FieldId field_id) {
auto it = runtime.ngram_indexings.find(field_id);
return it != runtime.ngram_indexings.end() && !it->second.empty();
}
void
ChunkedSegmentSealedImpl::SyncJsonNgramIndexState(
PublishedSegmentState& state,
const RuntimeResourceState& runtime,
FieldId field_id) {
if (RuntimeJsonNgramIndexReady(runtime, field_id)) {
set_bit(state.published_index_ready_bitset, field_id, true);
SetPublishedIndexRawDataInState(state, field_id, false);
return;
}
clear_bit_if_present(state.published_index_ready_bitset, field_id);
clear_bit_if_present(state.index_ready_bitset, field_id);
if (!get_bit_if_present(state.published_binlog_index_ready_bitset,
field_id)) {
ClearPublishedIndexRawDataInState(state, field_id);
ClearIndexRawDataInState(state, field_id);
}
}
std::shared_ptr<ChunkedSegmentSealedImpl::PublishedSegmentState>
ChunkedSegmentSealedImpl::ClonePublishedState(
const std::shared_ptr<const PublishedSegmentState>& current) const {
@@ -1114,6 +1285,16 @@ ChunkedSegmentSealedImpl::NormalizePublishedState(
state.index_has_raw_data[field_id] = raw_it->second;
}
}
for (const auto& [field_id, path_indexings] :
state.runtime->ngram_indexings) {
if (path_indexings.empty() ||
!field_exists_in_schema(state.schema, field_id)) {
continue;
}
set_bit(state.index_ready_bitset, field_id, true);
state.index_has_raw_data[field_id] = false;
}
}
for (size_t i = 0; i < state.published_index_ready_bitset.size(); ++i) {
@@ -1206,6 +1387,7 @@ ChunkedSegmentSealedImpl::FreezeRuntimeResourceState(
runtime->ngram_indexings = current.ngram_indexings;
runtime->text_lob_paths = current.text_lob_paths;
runtime->text_indexes = current.text_indexes;
runtime->json_indices = current.json_indices;
runtime->json_stats = current.json_stats;
runtime->reader = current.reader;
runtime->timestamps = current.timestamps;
@@ -3682,28 +3864,19 @@ void
ChunkedSegmentSealedImpl::DropJSONIndex(const FieldId field_id,
const std::string& nested_path) {
std::lock_guard<std::mutex> reopen_guard(reopen_mutex_);
{
std::unique_lock lck(mutex_);
json_indices.withWLock([&](auto& json_indexings) {
cancel_and_erase_json_indices(
json_indexings, field_id, nested_path);
});
}
auto next_runtime = CloneMutableRuntimeResourceState();
auto field_it = next_runtime->ngram_indexings.find(field_id);
if (field_it != next_runtime->ngram_indexings.end()) {
auto& path_indexings = field_it->second;
if (auto path_it = path_indexings.find(nested_path);
path_it != path_indexings.end()) {
cancel_warmup(path_it->second);
path_indexings.erase(path_it);
}
if (path_indexings.empty()) {
next_runtime->ngram_indexings.erase(field_it);
auto retired_indexings =
EraseJsonIndexesAtPath(*next_runtime, field_id, nested_path);
auto published_runtime = ToConstRuntimeState(std::move(next_runtime));
MutatePublishedStateLocked([&](PublishedSegmentState& state) {
state.runtime = published_runtime;
SyncJsonNgramIndexState(state, *published_runtime, field_id);
});
for (auto& indexing : retired_indexings) {
if (indexing != nullptr) {
indexing->CancelWarmup();
}
}
PublishRuntimeStateLocked(ToConstRuntimeState(std::move(next_runtime)));
}
void
@@ -4763,6 +4936,15 @@ ChunkedSegmentSealedImpl::ClearData() {
entry->indexing_->CancelWarmup();
}
}
for (const auto& json_index : runtime_snapshot->json_indices) {
cancel_warmup(json_index.index);
}
for (const auto& [_, path_indexings] :
runtime_snapshot->ngram_indexings) {
for (const auto& [__, indexing] : path_indexings) {
cancel_warmup(indexing);
}
}
}
{
std::unique_lock lck(mutex_);
@@ -4774,9 +4956,6 @@ ChunkedSegmentSealedImpl::ClearData() {
ngram_indexings_.withWLock([&](auto& ngram_indexings) {
cancel_and_clear_ngram_indexings(ngram_indexings);
});
json_indices.withWLock([&](auto& json_indexings) {
cancel_and_clear_json_indices(json_indexings);
});
insert_record_.clear();
timestamp_index_slot_.wlock()->reset();
pk_index_slot_.wlock()->reset();
@@ -5821,18 +6000,16 @@ ChunkedSegmentSealedImpl::HasIndex(FieldId field_id) const {
bool
ChunkedSegmentSealedImpl::HasJsonIndex(FieldId field_id) const {
// JSON indexes (JsonFlatIndex + JSON-cast) live in a separate per-segment
// vector rather than in either index bitset. Kept as a distinct API so
// HasIndex() preserves its narrower "scalar/vector/binlog index exists"
// semantics for ReorderConjunctExpr and other consumers.
return json_indices.withRLock([&](const auto& vec) {
for (const auto& index : vec) {
if (index.field_id == field_id) {
return true;
}
// JSON indexes (JsonFlatIndex + JSON-cast) remain distinct from the
// scalar/vector/binlog readiness bitsets, but their ownership now follows
// the published runtime snapshot.
auto runtime = CaptureRuntimeResourceState();
for (const auto& index : runtime->json_indices) {
if (index.field_id == field_id) {
return true;
}
return false;
});
}
return false;
}
bool
@@ -6807,6 +6984,21 @@ ChunkedSegmentSealedImpl::FinalizeLoadDiffForReopen(
}
}
CheckCancellation(op_ctx, id_, "ChunkedSegmentSealedImpl::ApplyLoadDiff()");
for (const auto& [field_id, nested_paths] : diff.json_indexes_to_drop) {
for (const auto& nested_path : nested_paths) {
committer.Commit([&, field_id, nested_path](
RuntimeResourceState& runtime,
PublishedSegmentState& staged_state) {
for (auto& retired :
EraseJsonIndexesAtPath(runtime, field_id, nested_path)) {
committer.RetireCacheIndexingLocked(std::move(retired));
}
SyncJsonNgramIndexState(staged_state, runtime, field_id);
});
}
}
CheckCancellation(op_ctx, id_, "ChunkedSegmentSealedImpl::ApplyLoadDiff()");
if (!diff.json_stats_to_drop.empty()) {
for (auto field_id : diff.json_stats_to_drop) {
@@ -184,6 +184,18 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
return {PinWrapper<const index::IndexBase*>(std::move(ca), index)};
}
std::vector<PinWrapper<const index::IndexBase*>>
PinJsonIndex(milvus::OpContext* op_ctx,
FieldId field_id,
const std::string& path,
DataType data_type,
bool any_type,
bool is_array) const override;
std::string
GetJsonFlatIndexNestedPath(FieldId field_id,
std::string_view query_path) const override;
bool
Contain(const PkType& pk) const override;
@@ -322,6 +334,13 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
std::shared_ptr<milvus::cachinglayer::CacheSlot<
milvus::index::TextMatchIndex>>>;
struct JsonIndex {
FieldId field_id;
std::string nested_path;
JsonCastType cast_type{JsonCastType::UNKNOWN};
index::CacheIndexBasePtr index;
};
// When non-zero commit_ts is active, every row in this segment carries it
// as its effective row timestamp (load-time overwrite). All timestamp
// consumers must route through this so the override applies uniformly on
@@ -347,6 +366,7 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
ngram_indexings;
std::unordered_map<FieldId, std::string> text_lob_paths;
std::unordered_map<FieldId, TextIndexVariant> text_indexes;
std::vector<JsonIndex> json_indices;
std::unordered_map<FieldId, std::shared_ptr<index::JsonKeyStats>>
json_stats;
std::shared_ptr<milvus_storage::api::Reader> reader;
@@ -1314,7 +1334,8 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
const SchemaPtr& schema_snapshot,
bool is_replace = false,
RuntimeResourceState* runtime = nullptr,
PublishedSegmentState* staged_state = nullptr);
PublishedSegmentState* staged_state = nullptr,
StagedStateCommitter* committer = nullptr);
void
LoadIndex(LoadIndexInfo& info, bool is_replace);
@@ -1408,6 +1429,30 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
static void
DropVectorIndexing(RuntimeResourceState& runtime, FieldId field_id);
static std::vector<index::CacheIndexBasePtr>
EraseJsonIndexings(RuntimeResourceState& runtime,
FieldId field_id,
std::string_view nested_path);
static index::CacheIndexBasePtr
EraseJsonNgramIndexing(RuntimeResourceState& runtime,
FieldId field_id,
std::string_view nested_path);
static std::vector<index::CacheIndexBasePtr>
EraseJsonIndexesAtPath(RuntimeResourceState& runtime,
FieldId field_id,
std::string_view nested_path);
static bool
RuntimeJsonNgramIndexReady(const RuntimeResourceState& runtime,
FieldId field_id);
static void
SyncJsonNgramIndexState(PublishedSegmentState& state,
const RuntimeResourceState& runtime,
FieldId field_id);
std::shared_ptr<PublishedSegmentState>
BuildNextPublishedState(
const std::shared_ptr<const PublishedSegmentState>& current,
@@ -1489,17 +1534,31 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
Publish(const std::shared_ptr<const PublishedSegmentState>& current,
const StateDelta& delta) {
std::vector<SealedIndexingEntryPtr> retired_indexings;
std::vector<index::CacheIndexBasePtr> retired_cache_indexings;
{
std::lock_guard<std::mutex> lock(mutex_);
segment_.PublishState(
segment_.BuildNextPublishedState(current, delta));
retired_indexings.swap(retired_vector_indexings_);
retired_cache_indexings.swap(retired_cache_indexings_);
}
for (auto& entry : retired_indexings) {
if (entry != nullptr && entry->indexing_ != nullptr) {
entry->indexing_->CancelWarmup();
}
}
for (auto& indexing : retired_cache_indexings) {
if (indexing != nullptr) {
indexing->CancelWarmup();
}
}
}
void
RetireCacheIndexingLocked(index::CacheIndexBasePtr indexing) {
if (indexing != nullptr) {
retired_cache_indexings_.push_back(std::move(indexing));
}
}
private:
@@ -1515,6 +1574,7 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
RuntimeResourceState* runtime_;
PublishedSegmentState* staged_state_;
std::vector<SealedIndexingEntryPtr> retired_vector_indexings_;
std::vector<index::CacheIndexBasePtr> retired_cache_indexings_;
std::mutex mutex_;
};
@@ -291,6 +291,17 @@ SegmentLoadInfo::ComputeDiffIndexes(LoadDiff& diff, SegmentLoadInfo& new_info) {
new_index_ids.insert(index_id);
}
}
std::unordered_map<FieldId, std::unordered_set<std::string>>
new_json_index_paths;
for (const auto& [field_id, index_paths] :
new_info.json_index_path_cache_) {
if (!new_info.HasFieldInSchema(field_id)) {
continue;
}
for (const auto& index_path : index_paths) {
new_json_index_paths[field_id].insert(index_path.second);
}
}
// Find indexes to load/replace: indexes in new_info but not in current
// Only consider fields that exist in the current schema (skip dropped fields)
for (const auto& [field_id, load_index_infos] :
@@ -321,6 +332,18 @@ SegmentLoadInfo::ComputeDiffIndexes(LoadDiff& diff, SegmentLoadInfo& new_info) {
for (auto index_id : index_ids) {
if (!new_info.HasFieldInSchema(field_id) ||
new_index_ids.find(index_id) == new_index_ids.end()) {
auto field_paths = json_index_path_cache_.find(field_id);
if (field_paths != json_index_path_cache_.end()) {
auto path = field_paths->second.find(index_id);
if (path != field_paths->second.end()) {
if (new_json_index_paths[field_id].find(path->second) ==
new_json_index_paths[field_id].end()) {
diff.json_indexes_to_drop[field_id].insert(
path->second);
}
continue;
}
}
diff.indexes_to_drop.insert(field_id);
}
}
+40 -1
View File
@@ -86,6 +86,12 @@ struct LoadDiff {
// Indexes that need to be dropped (field_id set)
std::set<FieldId> indexes_to_drop;
// JSON indexes that need to be dropped, keyed by field and nested path.
// JSON fields may have multiple indexes, so a field-only drop loses the
// identity needed to preserve sibling paths.
std::unordered_map<FieldId, std::unordered_set<std::string>>
json_indexes_to_drop;
// Field data that need to be dropped (field_id set)
// Only populated when both current and new use binlog mode
std::unordered_set<FieldId> field_data_to_drop;
@@ -136,7 +142,8 @@ struct LoadDiff {
!column_groups_to_lazyload.empty() ||
!column_groups_to_lazyreplace.empty() ||
!fields_to_reload.empty() || !indexes_to_drop.empty() ||
!field_data_to_drop.empty() || !fields_to_fill_default.empty() ||
!json_indexes_to_drop.empty() || !field_data_to_drop.empty() ||
!fields_to_fill_default.empty() ||
!text_indexes_to_load.empty() || !json_stats_to_load.empty() ||
!json_stats_to_replace.empty() || !json_stats_to_drop.empty() ||
!text_indexes_to_create.empty() || manifest_updated ||
@@ -289,6 +296,19 @@ struct LoadDiff {
}
oss << "], ";
// json_indexes_to_drop
oss << "json_indexes_to_drop=[";
first = true;
for (const auto& [field_id, paths] : json_indexes_to_drop) {
for (const auto& path : paths) {
if (!first)
oss << ", ";
first = false;
oss << field_id.get() << ":" << path;
}
}
oss << "], ";
// field_data_to_drop
oss << "field_data_to_drop=[";
first = true;
@@ -424,6 +444,7 @@ class SegmentLoadInfo {
schema_(other.schema_),
converted_field_index_cache_(other.converted_field_index_cache_),
field_index_id_cache_(other.field_index_id_cache_),
json_index_path_cache_(other.json_index_path_cache_),
field_index_has_raw_data_(other.field_index_has_raw_data_),
fields_filled_with_default_(other.fields_filled_with_default_),
column_groups_(other.column_groups_),
@@ -440,6 +461,7 @@ class SegmentLoadInfo {
converted_field_index_cache_(
std::move(other.converted_field_index_cache_)),
field_index_id_cache_(std::move(other.field_index_id_cache_)),
json_index_path_cache_(std::move(other.json_index_path_cache_)),
field_index_has_raw_data_(std::move(other.field_index_has_raw_data_)),
fields_filled_with_default_(
std::move(other.fields_filled_with_default_)),
@@ -460,6 +482,7 @@ class SegmentLoadInfo {
schema_ = other.schema_;
converted_field_index_cache_ = other.converted_field_index_cache_;
field_index_id_cache_ = other.field_index_id_cache_;
json_index_path_cache_ = other.json_index_path_cache_;
field_index_has_raw_data_ = other.field_index_has_raw_data_;
column_groups_ = other.column_groups_;
fields_filled_with_default_ = other.fields_filled_with_default_;
@@ -480,6 +503,7 @@ class SegmentLoadInfo {
converted_field_index_cache_ =
std::move(other.converted_field_index_cache_);
field_index_id_cache_ = std::move(other.field_index_id_cache_);
json_index_path_cache_ = std::move(other.json_index_path_cache_);
field_index_has_raw_data_ =
std::move(other.field_index_has_raw_data_);
fields_filled_with_default_ =
@@ -1167,6 +1191,7 @@ class SegmentLoadInfo {
prune_map(converted_field_index_cache_);
prune_map(field_index_id_cache_);
prune_map(json_index_path_cache_);
prune_set(field_index_has_raw_data_);
prune_set(fields_filled_with_default_);
prune_set(created_text_indexes_);
@@ -1190,6 +1215,7 @@ class SegmentLoadInfo {
// Convert index infos to LoadIndexInfo and build per-field cache
converted_field_index_cache_.clear();
field_index_id_cache_.clear();
json_index_path_cache_.clear();
field_index_has_raw_data_.clear();
for (int i = 0; i < info_.index_infos_size(); i++) {
const auto& index_info = info_.index_infos(i);
@@ -1225,6 +1251,13 @@ class SegmentLoadInfo {
if (CheckIndexHasRawData(load_index_info)) {
field_index_has_raw_data_.insert(field_id);
}
if (load_index_info.field_type == DataType::JSON) {
auto path_it = load_index_info.index_params.find(JSON_PATH);
if (path_it != load_index_info.index_params.end()) {
json_index_path_cache_[field_id][index_info.indexid()] =
path_it->second;
}
}
converted_field_index_cache_[field_id].push_back(
std::move(load_index_info));
}
@@ -1267,6 +1300,12 @@ class SegmentLoadInfo {
// needs to know which index ids are already present per field.
std::unordered_map<FieldId, std::vector<int64_t>> field_index_id_cache_;
// Lightweight JSON index identity retained after manifest load-info
// compaction so reopen can drop one nested path without affecting sibling
// indexes on the same JSON field.
std::unordered_map<FieldId, std::unordered_map<int64_t, std::string>>
json_index_path_cache_;
// set of field ids that corresponding index has raw data
std::set<FieldId> field_index_has_raw_data_;
@@ -2323,6 +2323,55 @@ TEST_F(SegmentLoadInfoTest, ComputeDiffIndexReplaceDropConflict) {
EXPECT_TRUE(actually_dropped.empty());
}
TEST_F(SegmentLoadInfoTest, ComputeDiffDropsJsonIndexByNestedPath) {
auto add_json_index = [](proto::segcore::SegmentLoadInfo& proto,
int64_t index_id,
const std::string& nested_path) {
auto* index = proto.add_index_infos();
index->set_fieldid(102);
index->set_indexid(index_id);
index->add_index_file_paths("/path/to/json_index_" +
std::to_string(index_id));
auto* type = index->add_index_params();
type->set_key(milvus::index::INDEX_TYPE);
type->set_value(milvus::index::INVERTED_INDEX_TYPE);
auto* path = index->add_index_params();
path->set_key(JSON_PATH);
path->set_value(nested_path);
auto* cast = index->add_index_params();
cast->set_key(JSON_CAST_TYPE);
cast->set_value("DOUBLE");
};
proto::segcore::SegmentLoadInfo current_proto;
current_proto.set_segmentid(100);
current_proto.set_num_of_rows(1000);
current_proto.set_manifest_path("/path/to/manifest");
add_json_index(current_proto, 5001, "a");
add_json_index(current_proto, 5002, "b");
proto::segcore::SegmentLoadInfo new_proto;
new_proto.set_segmentid(100);
new_proto.set_num_of_rows(1000);
new_proto.set_manifest_path("/path/to/manifest");
add_json_index(new_proto, 5002, "b");
SegmentLoadInfo current_info(current_proto, schema_);
SegmentLoadInfo new_info(new_proto, schema_);
current_info.SetColumnGroupsForTesting(
std::make_shared<milvus_storage::api::ColumnGroups>());
new_info.SetColumnGroupsForTesting(
std::make_shared<milvus_storage::api::ColumnGroups>());
current_info.CompactRuntimeInfoForManifest();
EXPECT_EQ(current_info.GetIndexInfoCount(), 0);
auto diff = current_info.ComputeDiff(new_info);
ASSERT_EQ(diff.json_indexes_to_drop.count(FieldId(102)), 1);
EXPECT_EQ(diff.json_indexes_to_drop.at(FieldId(102)).count("a"), 1);
EXPECT_EQ(diff.json_indexes_to_drop.at(FieldId(102)).count("b"), 0);
EXPECT_EQ(diff.indexes_to_drop.count(FieldId(102)), 0);
}
TEST_F(SegmentLoadInfoTest, ComputeDiffBinlogReplace) {
// When a field's binlog group changes, it goes to binlogs_to_replace
-116
View File
@@ -59,112 +59,6 @@ class SegmentSealed : public SegmentInternalInterface {
virtual InsertRecord<true>&
get_insert_record() = 0;
virtual std::vector<PinWrapper<const index::IndexBase*>>
PinJsonIndex(milvus::OpContext* op_ctx,
FieldId field_id,
const std::string& path,
DataType data_type,
bool any_type,
bool is_array) const override {
int path_len_diff = std::numeric_limits<int>::max();
index::CacheIndexBasePtr best_match = nullptr;
std::string_view path_view = path;
auto res = json_indices.withRLock(
[&](auto vec) -> PinWrapper<const index::IndexBase*> {
for (const auto& index : vec) {
if (index.field_id != field_id) {
continue;
}
switch (index.cast_type.data_type()) {
case JsonCastType::DataType::JSON:
if (path_view.length() <
index.nested_path.length()) {
continue;
}
if (path_view.substr(0,
index.nested_path.length()) ==
index.nested_path) {
int current_len_diff =
path_view.length() -
index.nested_path.length();
if (current_len_diff < path_len_diff) {
path_len_diff = current_len_diff;
best_match = index.index;
}
if (path_len_diff == 0) {
break;
}
}
break;
default:
if (index.nested_path != path) {
continue;
}
if (any_type) {
best_match = index.index;
break;
}
if (milvus::index::json::IsDataTypeSupported(
index.cast_type, data_type, is_array)) {
best_match = index.index;
break;
}
}
}
if (best_match == nullptr) {
return nullptr;
}
auto ca = SemiInlineGet(best_match->PinCells(op_ctx, {0}));
auto index = ca->get_cell_of(0);
return PinWrapper<const index::IndexBase*>(std::move(ca),
index);
});
if (res.get() == nullptr) {
return {};
}
return {res};
}
// Mirror of the JsonFlatIndex prefix-matching loop in PinJsonIndex(),
// but only reads the nested_path metadata and returns it -- does not
// touch the CacheSlot or pin any cell. Callers can use this to decide
// whether a JSON query is compatible with the indexed path before
// committing to the ScalarIndex exec path (and paying for a pin).
std::string
GetJsonFlatIndexNestedPath(FieldId field_id,
std::string_view query_path) const override {
return json_indices.withRLock([&](auto& vec) -> std::string {
std::string best_path;
int path_len_diff = std::numeric_limits<int>::max();
for (const auto& index : vec) {
if (index.field_id != field_id) {
continue;
}
if (index.cast_type.data_type() !=
JsonCastType::DataType::JSON) {
continue;
}
if (query_path.length() < index.nested_path.length()) {
continue;
}
if (query_path.substr(0, index.nested_path.length()) !=
index.nested_path) {
continue;
}
int current_len_diff =
query_path.length() - index.nested_path.length();
if (current_len_diff < path_len_diff) {
path_len_diff = current_len_diff;
best_path = index.nested_path;
}
if (path_len_diff == 0) {
break;
}
}
return best_path;
});
}
virtual PinWrapper<index::NgramInvertedIndex*>
GetNgramIndex(milvus::OpContext* op_ctx,
FieldId field_id) const override = 0;
@@ -178,16 +72,6 @@ class SegmentSealed : public SegmentInternalInterface {
type() const override {
return SegmentType::Sealed;
}
protected:
struct JsonIndex {
FieldId field_id;
std::string nested_path;
JsonCastType cast_type{JsonCastType::UNKNOWN};
index::CacheIndexBasePtr index;
};
folly::Synchronized<std::vector<JsonIndex>> json_indices;
};
using SegmentSealedSPtr = std::shared_ptr<SegmentSealed>;
-39
View File
@@ -788,45 +788,6 @@ LoadDeletedRecord(CSegmentInterface c_segment,
}
}
CStatus
UpdateSealedSegmentIndex(CSegmentInterface c_segment,
CLoadIndexInfo c_load_index_info) {
SCOPE_CGO_CALL_METRIC();
try {
auto segment_interface =
reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
auto segment =
dynamic_cast<milvus::segcore::SegmentSealed*>(segment_interface);
AssertInfo(segment != nullptr, "segment conversion failed");
auto load_index_info =
static_cast<milvus::segcore::LoadIndexInfo*>(c_load_index_info);
segment->LoadIndex(*load_index_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(&e);
}
}
CStatus
UpdateFieldRawDataSize(CSegmentInterface c_segment,
int64_t field_id,
int64_t num_rows,
int64_t field_data_size) {
SCOPE_CGO_CALL_METRIC();
try {
auto segment_interface =
reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
AssertInfo(segment_interface != nullptr, "segment conversion failed");
segment_interface->set_field_avg_size(
milvus::FieldId(field_id), num_rows, field_data_size);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(&e);
}
}
CStatus
DropFieldData(CSegmentInterface c_segment, int64_t field_id) {
SCOPE_CGO_CALL_METRIC();
-10
View File
@@ -244,16 +244,6 @@ CStatus
LoadDeletedRecord(CSegmentInterface c_segment,
CLoadDeletedRecordInfo deleted_record_info);
CStatus
UpdateSealedSegmentIndex(CSegmentInterface c_segment,
CLoadIndexInfo c_load_index_info);
CStatus
UpdateFieldRawDataSize(CSegmentInterface c_segment,
int64_t field_id,
int64_t num_rows,
int64_t field_data_size);
// This function is currently used only in test.
// Current implement supports only dropping of non-system fields.
CStatus
+1 -2
View File
@@ -1450,8 +1450,7 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
auto segment = CreateSealedWithFieldDataLoaded(schema, dataset);
// load vec index
status = UpdateSealedSegmentIndex(segment.get(), &load_index_info);
ASSERT_EQ(status.error_code, Success);
segment->LoadIndex(load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = CSearch(segment.get(),
+272
View File
@@ -4702,6 +4702,278 @@ TEST(SealedSegmentCowState, ClearPublishedStateDropsRuntimeSnapshot) {
EXPECT_FALSE(GetFieldBit(after->field_data_ready_bitset, payload));
}
TEST(SealedSegmentCowState, JsonIndexStagesAndFollowsSnapshotLifetime) {
auto schema = std::make_shared<Schema>();
auto pk = schema->AddDebugField("pk", DataType::INT64);
auto json = schema->AddDebugField("payload", DataType::JSON);
schema->set_primary_field_id(pk);
auto segment = CreateSealedSegment(schema);
auto* sealed = dynamic_cast<ChunkedSegmentSealedImpl*>(segment.get());
ASSERT_NE(sealed, nullptr);
std::array<int64_t, 4> values = {1, 2, 3, 4};
auto indexing = GenScalarIndexing<int64_t>(values.size(), values.data());
LoadIndexInfo load_info;
load_info.field_id = json.get();
load_info.field_type = DataType::JSON;
load_info.index_params = {
{index::INDEX_TYPE, index::INVERTED_INDEX_TYPE},
{JSON_PATH, "a"},
{JSON_CAST_TYPE, "DOUBLE"},
};
load_info.cache_index =
CreateTestCacheIndex("json-runtime", std::move(indexing));
auto loaded_index = load_info.cache_index;
auto current = sealed->TestGetPublishedStateSnapshot();
ASSERT_TRUE(current->runtime->json_indices.empty());
auto runtime = sealed->TestCloneMutableRuntimeResourceState();
ChunkedSegmentSealedImpl::StateDelta initial_delta;
initial_delta.schema = current->schema;
initial_delta.load_info = current->load_info;
initial_delta.runtime = sealed->TestFreezeRuntimeResourceState(runtime);
initial_delta.commit_ts = current->commit_ts;
auto staged = sealed->TestBuildNextPublishedState(current, initial_delta);
ChunkedSegmentSealedImpl::StateDelta final_delta;
final_delta.schema = current->schema;
final_delta.load_info = current->load_info;
final_delta.commit_ts = current->commit_ts;
sealed->TestStageLoadIndexThenPublish(
load_info,
false,
schema,
runtime,
staged.get(),
current,
final_delta,
[&] {
EXPECT_EQ(runtime->json_indices.size(), 1);
EXPECT_TRUE(current->runtime->json_indices.empty());
EXPECT_FALSE(sealed->HasJsonIndex(json));
});
auto published = sealed->TestGetPublishedStateSnapshot();
ASSERT_EQ(published->runtime->json_indices.size(), 1);
EXPECT_EQ(published->runtime->json_indices.front().index, loaded_index);
EXPECT_TRUE(sealed->HasJsonIndex(json));
sealed->DropJSONIndex(json, "a");
auto after_drop = sealed->TestGetPublishedStateSnapshot();
EXPECT_TRUE(after_drop->runtime->json_indices.empty());
EXPECT_FALSE(sealed->HasJsonIndex(json));
ASSERT_EQ(published->runtime->json_indices.size(), 1);
EXPECT_EQ(published->runtime->json_indices.front().index, loaded_index);
}
TEST(SealedSegmentCowState, JsonIndexReplaceScalarWithNgramErasesScalarPath) {
auto schema = std::make_shared<Schema>();
auto pk = schema->AddDebugField("pk", DataType::INT64);
auto json = schema->AddDebugField("payload", DataType::JSON);
schema->set_primary_field_id(pk);
auto segment = CreateSealedSegment(schema);
auto* sealed = dynamic_cast<ChunkedSegmentSealedImpl*>(segment.get());
ASSERT_NE(sealed, nullptr);
std::array<int64_t, 4> values = {1, 2, 3, 4};
auto make_load_info = [&](std::string key,
std::string path,
bool is_ngram) {
LoadIndexInfo info;
info.field_id = json.get();
info.field_type = DataType::JSON;
info.index_params = {
{index::INDEX_TYPE,
is_ngram ? index::NGRAM_INDEX_TYPE : index::INVERTED_INDEX_TYPE},
{JSON_PATH, std::move(path)},
};
if (!is_ngram) {
info.index_params[JSON_CAST_TYPE] = "DOUBLE";
}
auto indexing =
GenScalarIndexing<int64_t>(values.size(), values.data());
info.cache_index =
CreateTestCacheIndex(std::move(key), std::move(indexing));
return info;
};
auto sibling = make_load_info("json-scalar-b", "b", false);
auto sibling_index = sibling.cache_index;
sealed->LoadIndex(sibling);
auto original = make_load_info("json-scalar-a", "a", false);
auto original_index = original.cache_index;
sealed->LoadIndex(original);
auto current = sealed->TestGetPublishedStateSnapshot();
ASSERT_EQ(current->runtime->json_indices.size(), 2);
EXPECT_TRUE(current->runtime->ngram_indexings.empty());
EXPECT_FALSE(GetFieldBit(current->index_ready_bitset, json));
auto runtime = sealed->TestCloneMutableRuntimeResourceState();
ChunkedSegmentSealedImpl::StateDelta initial_delta;
initial_delta.schema = current->schema;
initial_delta.load_info = current->load_info;
initial_delta.runtime = sealed->TestFreezeRuntimeResourceState(runtime);
initial_delta.commit_ts = current->commit_ts;
auto staged = sealed->TestBuildNextPublishedState(current, initial_delta);
auto replacement = make_load_info("json-ngram-a", "a", true);
auto replacement_index = replacement.cache_index;
ChunkedSegmentSealedImpl::StateDelta final_delta;
final_delta.schema = current->schema;
final_delta.load_info = current->load_info;
final_delta.commit_ts = current->commit_ts;
sealed->TestStageLoadIndexThenPublish(
replacement,
true,
schema,
runtime,
staged.get(),
current,
final_delta,
[&] {
ASSERT_EQ(runtime->json_indices.size(), 1);
EXPECT_EQ(runtime->json_indices.front().nested_path, "b");
EXPECT_EQ(runtime->json_indices.front().index, sibling_index);
ASSERT_EQ(runtime->ngram_indexings.count(json), 1);
ASSERT_EQ(runtime->ngram_indexings.at(json).count("a"), 1);
EXPECT_EQ(runtime->ngram_indexings.at(json).at("a"),
replacement_index);
EXPECT_TRUE(GetFieldBit(staged->index_ready_bitset, json));
ASSERT_EQ(current->runtime->json_indices.size(), 2);
EXPECT_TRUE(current->runtime->ngram_indexings.empty());
});
auto published = sealed->TestGetPublishedStateSnapshot();
ASSERT_EQ(published->runtime->json_indices.size(), 1);
EXPECT_EQ(published->runtime->json_indices.front().nested_path, "b");
EXPECT_EQ(published->runtime->json_indices.front().index, sibling_index);
ASSERT_EQ(published->runtime->ngram_indexings.count(json), 1);
EXPECT_EQ(published->runtime->ngram_indexings.at(json).at("a"),
replacement_index);
EXPECT_TRUE(GetFieldBit(published->index_ready_bitset, json));
ASSERT_EQ(current->runtime->json_indices.size(), 2);
auto old_path = std::find_if(
current->runtime->json_indices.begin(),
current->runtime->json_indices.end(),
[](const auto& index) { return index.nested_path == "a"; });
ASSERT_NE(old_path, current->runtime->json_indices.end());
EXPECT_EQ(old_path->index, original_index);
}
TEST(SealedSegmentCowState, JsonIndexReplaceNgramWithScalarErasesNgramPath) {
auto schema = std::make_shared<Schema>();
auto pk = schema->AddDebugField("pk", DataType::INT64);
auto json = schema->AddDebugField("payload", DataType::JSON);
schema->set_primary_field_id(pk);
auto segment = CreateSealedSegment(schema);
auto* sealed = dynamic_cast<ChunkedSegmentSealedImpl*>(segment.get());
ASSERT_NE(sealed, nullptr);
std::array<int64_t, 4> values = {1, 2, 3, 4};
auto make_load_info = [&](std::string key,
std::string path,
bool is_ngram) {
LoadIndexInfo info;
info.field_id = json.get();
info.field_type = DataType::JSON;
info.index_params = {
{index::INDEX_TYPE,
is_ngram ? index::NGRAM_INDEX_TYPE : index::INVERTED_INDEX_TYPE},
{JSON_PATH, std::move(path)},
};
if (!is_ngram) {
info.index_params[JSON_CAST_TYPE] = "DOUBLE";
}
auto indexing =
GenScalarIndexing<int64_t>(values.size(), values.data());
info.cache_index =
CreateTestCacheIndex(std::move(key), std::move(indexing));
return info;
};
auto sibling = make_load_info("json-scalar-b", "b", false);
auto sibling_index = sibling.cache_index;
sealed->LoadIndex(sibling);
auto original = make_load_info("json-ngram-a", "a", true);
auto original_index = original.cache_index;
sealed->LoadIndex(original);
auto current = sealed->TestGetPublishedStateSnapshot();
ASSERT_EQ(current->runtime->json_indices.size(), 1);
ASSERT_EQ(current->runtime->ngram_indexings.count(json), 1);
EXPECT_EQ(current->runtime->ngram_indexings.at(json).at("a"),
original_index);
EXPECT_TRUE(GetFieldBit(current->index_ready_bitset, json));
auto runtime = sealed->TestCloneMutableRuntimeResourceState();
ChunkedSegmentSealedImpl::StateDelta initial_delta;
initial_delta.schema = current->schema;
initial_delta.load_info = current->load_info;
initial_delta.runtime = sealed->TestFreezeRuntimeResourceState(runtime);
initial_delta.commit_ts = current->commit_ts;
auto staged = sealed->TestBuildNextPublishedState(current, initial_delta);
auto replacement = make_load_info("json-scalar-a", "a", false);
auto replacement_index = replacement.cache_index;
ChunkedSegmentSealedImpl::StateDelta final_delta;
final_delta.schema = current->schema;
final_delta.load_info = current->load_info;
final_delta.commit_ts = current->commit_ts;
sealed->TestStageLoadIndexThenPublish(
replacement,
true,
schema,
runtime,
staged.get(),
current,
final_delta,
[&] {
EXPECT_TRUE(runtime->ngram_indexings.empty());
ASSERT_EQ(runtime->json_indices.size(), 2);
auto replacement_path = std::find_if(
runtime->json_indices.begin(),
runtime->json_indices.end(),
[](const auto& index) { return index.nested_path == "a"; });
ASSERT_NE(replacement_path, runtime->json_indices.end());
EXPECT_EQ(replacement_path->index, replacement_index);
EXPECT_FALSE(GetFieldBit(staged->index_ready_bitset, json));
ASSERT_EQ(current->runtime->ngram_indexings.count(json), 1);
EXPECT_EQ(current->runtime->ngram_indexings.at(json).at("a"),
original_index);
});
auto published = sealed->TestGetPublishedStateSnapshot();
EXPECT_TRUE(published->runtime->ngram_indexings.empty());
ASSERT_EQ(published->runtime->json_indices.size(), 2);
auto sibling_path = std::find_if(
published->runtime->json_indices.begin(),
published->runtime->json_indices.end(),
[](const auto& index) { return index.nested_path == "b"; });
ASSERT_NE(sibling_path, published->runtime->json_indices.end());
EXPECT_EQ(sibling_path->index, sibling_index);
auto replacement_path = std::find_if(
published->runtime->json_indices.begin(),
published->runtime->json_indices.end(),
[](const auto& index) { return index.nested_path == "a"; });
ASSERT_NE(replacement_path, published->runtime->json_indices.end());
EXPECT_EQ(replacement_path->index, replacement_index);
EXPECT_FALSE(GetFieldBit(published->index_ready_bitset, json));
ASSERT_EQ(current->runtime->ngram_indexings.count(json), 1);
EXPECT_EQ(current->runtime->ngram_indexings.at(json).at("a"),
original_index);
}
TEST(SealedSegmentCowState, JsonStatsLivesInRuntimeSnapshot) {
auto schema = std::make_shared<Schema>();
auto pk = schema->AddDebugField("pk", DataType::INT64);
-4
View File
@@ -135,10 +135,6 @@ func packLoadSegmentRequest(
indexInfo []*indexpb.IndexInfo,
) *querypb.LoadSegmentsRequest {
loadScope := querypb.LoadScope_Full
if action.Type() == ActionTypeUpdate {
loadScope = querypb.LoadScope_Index
}
if action.Type() == ActionTypeStatsUpdate {
loadScope = querypb.LoadScope_Stats
}
-5
View File
@@ -573,11 +573,6 @@ func (s *UtilsSuite) TestPackLoadSegmentRequestLoadScope() {
action ActionType
wantScope querypb.LoadScope
}{
{
name: "index update",
action: ActionTypeUpdate,
wantScope: querypb.LoadScope_Index,
},
{
name: "legacy stats update",
action: ActionTypeStatsUpdate,
+3 -7
View File
@@ -6,14 +6,10 @@ import (
context "context"
commonpb "github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
internalpb "github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
mock "github.com/stretchr/testify/mock"
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
streamrpc "github.com/milvus-io/milvus/internal/util/streamrpc"
internalpb "github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
mock "github.com/stretchr/testify/mock"
)
// MockWorker is an autogenerated mock type for the Worker type
@@ -692,10 +692,6 @@ func (sd *shardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSeg
return err
}
// load index segment need no stream delete and distribution change
if req.GetLoadScope() == querypb.LoadScope_Index {
return nil
}
if req.GetLoadScope() == querypb.LoadScope_Reopen {
return sd.handleReopenPostLoad(ctx, req)
}
-33
View File
@@ -141,39 +141,6 @@ func (node *QueryNode) loadDeltaLogs(ctx context.Context, req *querypb.LoadSegme
return merr.Success()
}
func (node *QueryNode) loadIndex(ctx context.Context, req *querypb.LoadSegmentsRequest) *commonpb.Status {
log := mlog.With(
mlog.FieldCollectionID(req.GetCollectionID()),
mlog.Int64s("segmentIDs", lo.Map(req.GetInfos(), func(info *querypb.SegmentLoadInfo, _ int) int64 { return info.GetSegmentID() })),
)
status := merr.Success()
log.Info(ctx, "start to load index")
for _, info := range req.GetInfos() {
log := mlog.With(mlog.FieldSegmentID(info.GetSegmentID()))
segment := node.manager.Segment.GetSealed(info.GetSegmentID())
if segment == nil {
log.Warn(ctx, "segment not found for load index operation")
continue
}
localSegment, ok := segment.(*segments.LocalSegment)
if !ok {
log.Warn(ctx, "segment not local for load index opeartion")
continue
}
err := node.loader.LoadIndex(ctx, localSegment, info, req.Version)
if err != nil {
log.Warn(ctx, "failed to load index", mlog.Err(err))
status = merr.Status(err)
break
}
}
return status
}
func (node *QueryNode) reopenSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) *commonpb.Status {
log := mlog.With(
mlog.FieldCollectionID(req.GetCollectionID()),
@@ -27,15 +27,11 @@ import "C"
import (
"context"
"runtime"
"time"
"unsafe"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus/pkg/v3/metrics"
"github.com/milvus-io/milvus/pkg/v3/proto/cgopb"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
)
// LoadIndexInfo is a wrapper of the underlying C-structure C.CLoadIndexInfo
@@ -66,15 +62,6 @@ func deleteLoadIndexInfo(info *LoadIndexInfo) {
}).Await()
}
func (li *LoadIndexInfo) cleanLocalData(ctx context.Context) error {
var status C.CStatus
GetDynamicPool().Submit(func() (any, error) {
status = C.CleanLoadedIndex(li.cLoadIndexInfo)
return nil, nil
}).Await()
return HandleCStatus(ctx, &status, "failed to clean cached data on disk")
}
func (li *LoadIndexInfo) appendLoadIndexInfo(ctx context.Context, info *cgopb.LoadIndexInfo) error {
marshaled, err := proto.Marshal(info)
if err != nil {
@@ -98,23 +85,3 @@ func (li *LoadIndexInfo) setShard(shard string) {
defer C.free(unsafe.Pointer(cShard))
C.SetLoadIndexInfoShard(li.cLoadIndexInfo, cShard)
}
func (li *LoadIndexInfo) loadIndex(ctx context.Context) error {
var status C.CStatus
_, _ = GetLoadPool().Submit(func() (any, error) {
start := time.Now()
defer func() {
metrics.QueryNodeCGOCallLatency.WithLabelValues(
paramtable.GetStringNodeID(),
"AppendIndexV2",
"Sync",
).Observe(float64(time.Since(start).Milliseconds()))
}()
traceCtx := ParseCTraceContext(ctx)
status = C.AppendIndexV2(traceCtx.ctx, li.cLoadIndexInfo)
runtime.KeepAlive(traceCtx)
return nil, nil
}).Await()
return HandleCStatus(ctx, &status, "AppendIndexV2 failed")
}
+2 -55
View File
@@ -6,14 +6,10 @@ import (
context "context"
commonpb "github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
mock "github.com/stretchr/testify/mock"
pkoracle "github.com/milvus-io/milvus/internal/querynodev2/pkoracle"
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
storage "github.com/milvus-io/milvus/internal/storage"
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
mock "github.com/stretchr/testify/mock"
)
// MockLoader is an autogenerated mock type for the Loader type
@@ -274,55 +270,6 @@ func (_c *MockLoader_LoadDeltaLogs_Call) RunAndReturn(run func(context.Context,
return _c
}
// LoadIndex provides a mock function with given fields: ctx, segment, info, version
func (_m *MockLoader) LoadIndex(ctx context.Context, segment Segment, info *querypb.SegmentLoadInfo, version int64) error {
ret := _m.Called(ctx, segment, info, version)
if len(ret) == 0 {
panic("no return value specified for LoadIndex")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, Segment, *querypb.SegmentLoadInfo, int64) error); ok {
r0 = rf(ctx, segment, info, version)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockLoader_LoadIndex_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadIndex'
type MockLoader_LoadIndex_Call struct {
*mock.Call
}
// LoadIndex is a helper method to define mock.On call
// - ctx context.Context
// - segment Segment
// - info *querypb.SegmentLoadInfo
// - version int64
func (_e *MockLoader_Expecter) LoadIndex(ctx interface{}, segment interface{}, info interface{}, version interface{}) *MockLoader_LoadIndex_Call {
return &MockLoader_LoadIndex_Call{Call: _e.mock.On("LoadIndex", ctx, segment, info, version)}
}
func (_c *MockLoader_LoadIndex_Call) Run(run func(ctx context.Context, segment Segment, info *querypb.SegmentLoadInfo, version int64)) *MockLoader_LoadIndex_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(Segment), args[2].(*querypb.SegmentLoadInfo), args[3].(int64))
})
return _c
}
func (_c *MockLoader_LoadIndex_Call) Return(_a0 error) *MockLoader_LoadIndex_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockLoader_LoadIndex_Call) RunAndReturn(run func(context.Context, Segment, *querypb.SegmentLoadInfo, int64) error) *MockLoader_LoadIndex_Call {
_c.Call.Return(run)
return _c
}
// ReopenSegments provides a mock function with given fields: ctx, loadInfos
func (_m *MockLoader) ReopenSegments(ctx context.Context, loadInfos []*querypb.SegmentLoadInfo) error {
ret := _m.Called(ctx, loadInfos)
@@ -6,10 +6,8 @@ import (
context "context"
commonpb "github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
mock "github.com/stretchr/testify/mock"
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
mock "github.com/stretchr/testify/mock"
)
// MockSegmentManager is an autogenerated mock type for the SegmentManager type
@@ -753,6 +751,40 @@ func (_c *MockSegmentManager_Put_Call) RunAndReturn(run func(context.Context, co
return _c
}
// ReleaseDetached provides a mock function with given fields: ctx, segment
func (_m *MockSegmentManager) ReleaseDetached(ctx context.Context, segment Segment) {
_m.Called(ctx, segment)
}
// MockSegmentManager_ReleaseDetached_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseDetached'
type MockSegmentManager_ReleaseDetached_Call struct {
*mock.Call
}
// ReleaseDetached is a helper method to define mock.On call
// - ctx context.Context
// - segment Segment
func (_e *MockSegmentManager_Expecter) ReleaseDetached(ctx interface{}, segment interface{}) *MockSegmentManager_ReleaseDetached_Call {
return &MockSegmentManager_ReleaseDetached_Call{Call: _e.mock.On("ReleaseDetached", ctx, segment)}
}
func (_c *MockSegmentManager_ReleaseDetached_Call) Run(run func(ctx context.Context, segment Segment)) *MockSegmentManager_ReleaseDetached_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(Segment))
})
return _c
}
func (_c *MockSegmentManager_ReleaseDetached_Call) Return() *MockSegmentManager_ReleaseDetached_Call {
_c.Call.Return()
return _c
}
func (_c *MockSegmentManager_ReleaseDetached_Call) RunAndReturn(run func(context.Context, Segment)) *MockSegmentManager_ReleaseDetached_Call {
_c.Run(run)
return _c
}
// Remove provides a mock function with given fields: ctx, segmentID, scope
func (_m *MockSegmentManager) Remove(ctx context.Context, segmentID int64, scope querypb.DataScope) (int, int) {
ret := _m.Called(ctx, segmentID, scope)
@@ -811,98 +843,6 @@ func (_c *MockSegmentManager_Remove_Call) RunAndReturn(run func(context.Context,
return _c
}
// Detach provides a mock function with given fields: ctx, segmentID, scope
func (_m *MockSegmentManager) Detach(ctx context.Context, segmentID int64, scope querypb.DataScope) (int, int) {
ret := _m.Called(ctx, segmentID, scope)
if len(ret) == 0 {
panic("no return value specified for Detach")
}
var r0 int
var r1 int
if rf, ok := ret.Get(0).(func(context.Context, int64, querypb.DataScope) (int, int)); ok {
return rf(ctx, segmentID, scope)
}
if rf, ok := ret.Get(0).(func(context.Context, int64, querypb.DataScope) int); ok {
r0 = rf(ctx, segmentID, scope)
} else {
r0 = ret.Get(0).(int)
}
if rf, ok := ret.Get(1).(func(context.Context, int64, querypb.DataScope) int); ok {
r1 = rf(ctx, segmentID, scope)
} else {
r1 = ret.Get(1).(int)
}
return r0, r1
}
// MockSegmentManager_Detach_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Detach'
type MockSegmentManager_Detach_Call struct {
*mock.Call
}
// Detach is a helper method to define mock.On call
// - ctx context.Context
// - segmentID int64
// - scope querypb.DataScope
func (_e *MockSegmentManager_Expecter) Detach(ctx interface{}, segmentID interface{}, scope interface{}) *MockSegmentManager_Detach_Call {
return &MockSegmentManager_Detach_Call{Call: _e.mock.On("Detach", ctx, segmentID, scope)}
}
func (_c *MockSegmentManager_Detach_Call) Run(run func(ctx context.Context, segmentID int64, scope querypb.DataScope)) *MockSegmentManager_Detach_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64), args[2].(querypb.DataScope))
})
return _c
}
func (_c *MockSegmentManager_Detach_Call) Return(_a0 int, _a1 int) *MockSegmentManager_Detach_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockSegmentManager_Detach_Call) RunAndReturn(run func(context.Context, int64, querypb.DataScope) (int, int)) *MockSegmentManager_Detach_Call {
_c.Call.Return(run)
return _c
}
// ReleaseDetached provides a mock function with given fields: ctx, segment
func (_m *MockSegmentManager) ReleaseDetached(ctx context.Context, segment Segment) {
_m.Called(ctx, segment)
}
// MockSegmentManager_ReleaseDetached_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseDetached'
type MockSegmentManager_ReleaseDetached_Call struct {
*mock.Call
}
// ReleaseDetached is a helper method to define mock.On call
// - ctx context.Context
// - segment Segment
func (_e *MockSegmentManager_Expecter) ReleaseDetached(ctx interface{}, segment interface{}) *MockSegmentManager_ReleaseDetached_Call {
return &MockSegmentManager_ReleaseDetached_Call{Call: _e.mock.On("ReleaseDetached", ctx, segment)}
}
func (_c *MockSegmentManager_ReleaseDetached_Call) Run(run func(ctx context.Context, segment Segment)) *MockSegmentManager_ReleaseDetached_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(Segment))
})
return _c
}
func (_c *MockSegmentManager_ReleaseDetached_Call) Return() *MockSegmentManager_ReleaseDetached_Call {
_c.Call.Return()
return _c
}
func (_c *MockSegmentManager_ReleaseDetached_Call) RunAndReturn(run func(context.Context, Segment)) *MockSegmentManager_ReleaseDetached_Call {
_c.Run(run)
return _c
}
// RemoveBy provides a mock function with given fields: ctx, filters
func (_m *MockSegmentManager) RemoveBy(ctx context.Context, filters ...SegmentFilter) (int, int) {
_va := make([]interface{}, len(filters))
-147
View File
@@ -1242,153 +1242,6 @@ func GetCLoadInfoWithFunc(ctx context.Context,
return f(loadIndexInfo)
}
func (s *LocalSegment) LoadIndex(ctx context.Context, indexInfo *querypb.FieldIndexInfo, fieldType schemapb.DataType) error {
log := mlog.With(
mlog.FieldCollectionID(s.Collection()),
mlog.FieldPartitionID(s.Partition()),
mlog.FieldSegmentID(s.ID()),
mlog.FieldFieldID(indexInfo.GetFieldID()),
mlog.FieldIndexID(indexInfo.GetIndexID()),
)
old := s.GetIndexByID(indexInfo.GetIndexID())
// the index loaded
if old != nil && old.IsLoaded {
log.Warn(ctx, "index already loaded")
return nil
}
ctx, sp := otel.Tracer(typeutil.QueryNodeRole).Start(ctx, fmt.Sprintf("LoadIndex-%d-%d", s.ID(), indexInfo.GetFieldID()))
defer sp.End()
tr := timerecord.NewTimeRecorder("loadIndex")
schemaHelper, err := typeutil.CreateSchemaHelper(s.GetCollection().Schema())
if err != nil {
return err
}
fieldSchema, err := schemaHelper.GetFieldFromID(indexInfo.GetFieldID())
if err != nil {
return err
}
// // if segment is pk sorted, user created indexes bring no performance gain but extra memory usage
if s.IsSorted() && fieldSchema.GetIsPrimaryKey() {
log.Info(ctx, "skip loading index for pk field in sorted segment")
// set field index, preventing repeated loading index task
s.fieldIndexes.Insert(indexInfo.GetFieldID(), &IndexedFieldInfo{
FieldBinlog: &datapb.FieldBinlog{
FieldID: indexInfo.GetFieldID(),
},
IndexInfo: indexInfo,
IsLoaded: true,
})
return nil
}
return s.innerLoadIndex(ctx, fieldSchema, indexInfo, tr, fieldType)
}
func (s *LocalSegment) innerLoadIndex(ctx context.Context,
fieldSchema *schemapb.FieldSchema,
indexInfo *querypb.FieldIndexInfo,
tr *timerecord.TimeRecorder,
fieldType schemapb.DataType,
) error {
err := GetCLoadInfoWithFunc(ctx, fieldSchema,
s.LoadInfo(), indexInfo, func(loadIndexInfo *LoadIndexInfo) error {
newLoadIndexInfoSpan := tr.RecordSpan()
if err := loadIndexInfo.loadIndex(ctx); err != nil {
if loadIndexInfo.cleanLocalData(ctx) != nil {
mlog.Warn(ctx, "failed to clean cached data on disk after append index failed",
mlog.FieldBuildID(indexInfo.BuildID),
mlog.Int64("index version", indexInfo.IndexVersion))
}
return err
}
if s.Type() != SegmentTypeSealed {
return merr.WrapErrServiceInternalMsg("updateSegmentIndex failed, illegal segment type %s, segmentID = %d", s.segmentType, s.ID())
}
appendLoadIndexInfoSpan := tr.RecordSpan()
// 3.
err := s.UpdateIndexInfo(ctx, indexInfo, loadIndexInfo)
if err != nil {
return err
}
updateIndexInfoSpan := tr.RecordSpan()
mlog.Info(ctx, "Finish loading index",
mlog.Duration("newLoadIndexInfoSpan", newLoadIndexInfoSpan),
mlog.Duration("appendLoadIndexInfoSpan", appendLoadIndexInfoSpan),
mlog.Duration("updateIndexInfoSpan", updateIndexInfoSpan),
)
return nil
})
if err != nil {
mlog.Warn(ctx, "load index failed", mlog.Err(err))
}
return err
}
func (s *LocalSegment) UpdateIndexInfo(ctx context.Context, indexInfo *querypb.FieldIndexInfo, info *LoadIndexInfo) error {
log := mlog.With(
mlog.FieldCollectionID(s.Collection()),
mlog.FieldPartitionID(s.Partition()),
mlog.FieldSegmentID(s.ID()),
mlog.FieldFieldID(indexInfo.FieldID),
)
if !s.ptrLock.PinIf(state.IsNotReleased) {
return merr.WrapErrSegmentNotLoaded(s.ID(), "segment released")
}
defer s.ptrLock.Unpin()
var status C.CStatus
GetDynamicPool().Submit(func() (any, error) {
status = C.UpdateSealedSegmentIndex(s.ptr, info.cLoadIndexInfo)
return nil, nil
}).Await()
if err := HandleCStatus(ctx, &status, "UpdateSealedSegmentIndex failed",
mlog.FieldCollectionID(s.Collection()),
mlog.FieldPartitionID(s.Partition()),
mlog.FieldSegmentID(s.ID()),
mlog.FieldFieldID(indexInfo.FieldID)); err != nil {
return err
}
s.fieldIndexes.Insert(indexInfo.GetIndexID(), &IndexedFieldInfo{
FieldBinlog: &datapb.FieldBinlog{
FieldID: indexInfo.GetFieldID(),
},
IndexInfo: indexInfo,
IsLoaded: true,
})
log.Info(ctx, "updateSegmentIndex done")
return nil
}
func (s *LocalSegment) UpdateFieldRawDataSize(ctx context.Context, numRows int64, fieldBinlog *datapb.FieldBinlog) error {
var status C.CStatus
fieldID := fieldBinlog.FieldID
fieldDataSize := int64(0)
for _, binlog := range fieldBinlog.GetBinlogs() {
fieldDataSize += binlog.GetMemorySize()
}
GetDynamicPool().Submit(func() (any, error) {
status = C.UpdateFieldRawDataSize(s.ptr, C.int64_t(fieldID), C.int64_t(numRows), C.int64_t(fieldDataSize))
return nil, nil
}).Await()
if err := HandleCStatus(ctx, &status, "updateFieldRawDataSize failed"); err != nil {
return err
}
mlog.Info(ctx, "updateFieldRawDataSize done", mlog.FieldSegmentID(s.ID()))
return nil
}
func (s *LocalSegment) syncFieldJSONStatsFromLoadInfo(ctx context.Context, loadInfo *querypb.SegmentLoadInfo) {
jsonStatsInfo := make(map[int64]*querypb.JsonStatsInfo)
if !paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool() {
@@ -90,12 +90,6 @@ type Loader interface {
// GetChunkManager returns the chunk manager for remote storage access.
GetChunkManager() storage.ChunkManager
// LoadIndex append index for segment and remove vector binlogs.
LoadIndex(ctx context.Context,
segment Segment,
info *querypb.SegmentLoadInfo,
version int64) error
// ReopenSegments update segment data according to new load info.
ReopenSegments(ctx context.Context,
loadInfos []*querypb.SegmentLoadInfo,
@@ -1198,45 +1192,6 @@ func loadSealedSegmentFields(ctx context.Context, collection *Collection, segmen
return nil
}
func (loader *segmentLoader) loadFieldsIndex(ctx context.Context,
schemaHelper *typeutil.SchemaHelper,
segment *LocalSegment,
numRows int64,
indexedFieldInfos map[int64]*IndexedFieldInfo,
) error {
for _, fieldInfo := range indexedFieldInfos {
fieldID := fieldInfo.IndexInfo.FieldID
indexInfo := fieldInfo.IndexInfo
tr := timerecord.NewTimeRecorder("loadFieldIndex")
err := loader.loadFieldIndex(ctx, segment, indexInfo)
loadFieldIndexSpan := tr.RecordSpan()
if err != nil {
return err
}
mlog.Info(context.TODO(), "load field binlogs done for sealed segment with index",
mlog.Int64("fieldID", fieldID),
mlog.Any("binlog", fieldInfo.FieldBinlog.Binlogs),
mlog.Int32("current_index_version", fieldInfo.IndexInfo.GetCurrentIndexVersion()),
mlog.Duration("load_duration", loadFieldIndexSpan),
)
// set average row data size of variable field
field, err := schemaHelper.GetFieldFromID(fieldID)
if err != nil {
return err
}
if typeutil.IsVariableDataType(field.GetDataType()) {
err = segment.UpdateFieldRawDataSize(ctx, numRows, fieldInfo.FieldBinlog)
if err != nil {
return err
}
}
}
return nil
}
func (loader *segmentLoader) loadBm25Stats(ctx context.Context, segmentID int64, stats map[int64]*storage.BM25Stats, binlogPaths map[int64][]string) error {
if len(binlogPaths) == 0 {
mlog.Info(context.TODO(), "there are no bm25 stats logs saved with segment")
@@ -1279,29 +1234,6 @@ func (loader *segmentLoader) loadBm25Stats(ctx context.Context, segmentID int64,
return nil
}
func (loader *segmentLoader) loadFieldIndex(ctx context.Context, segment *LocalSegment, indexInfo *querypb.FieldIndexInfo) error {
filteredPaths := make([]string, 0, len(indexInfo.IndexFilePaths))
for _, indexPath := range indexInfo.IndexFilePaths {
if path.Base(indexPath) != storage.IndexParamsKey {
filteredPaths = append(filteredPaths, indexPath)
}
}
indexInfo.IndexFilePaths = filteredPaths
fieldType, err := loader.getFieldType(segment.Collection(), indexInfo.FieldID)
if err != nil {
return err
}
collection := loader.manager.Collection.Get(segment.Collection())
if collection == nil {
return merr.WrapErrCollectionNotLoaded(segment.Collection(), "failed to load field index")
}
return segment.LoadIndex(ctx, indexInfo, fieldType)
}
func (loader *segmentLoader) loadBloomFilter(
ctx context.Context,
segmentID int64,
@@ -2492,92 +2424,6 @@ func SupportInterimIndexDataType(dataType schemapb.DataType) bool {
dataType == schemapb.DataType_BFloat16Vector
}
func (loader *segmentLoader) getFieldType(collectionID, fieldID int64) (schemapb.DataType, error) {
collection := loader.manager.Collection.Get(collectionID)
if collection == nil {
return 0, merr.WrapErrCollectionNotFound(collectionID)
}
for _, field := range collection.Schema().GetFields() {
if field.GetFieldID() == fieldID {
return field.GetDataType(), nil
}
}
for _, structField := range collection.Schema().GetStructArrayFields() {
if structField.GetFieldID() == fieldID {
return schemapb.DataType_ArrayOfStruct, nil
}
for _, subField := range structField.GetFields() {
if subField.GetFieldID() == fieldID {
return subField.GetDataType(), nil
}
}
}
return 0, merr.WrapErrFieldNotFound(fieldID)
}
func (loader *segmentLoader) LoadIndex(ctx context.Context,
seg Segment,
loadInfo *querypb.SegmentLoadInfo,
version int64,
) error {
segment, ok := seg.(*LocalSegment)
if !ok {
return merr.WrapErrParameterInvalid("LocalSegment", fmt.Sprintf("%T", seg))
}
// Filter out LOADING segments only
// use None to avoid loaded check
infos := loader.prepare(ctx, commonpb.SegmentState_SegmentStateNone, loadInfo)
defer loader.unregister(infos...)
indexInfo := lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
info = typeutil.Clone(info)
// remain binlog paths whose field id is in index infos to estimate resource usage correctly
indexFields := typeutil.NewSet(lo.Map(info.GetIndexInfos(), func(indexInfo *querypb.FieldIndexInfo, _ int) int64 { return indexInfo.GetFieldID() })...)
var binlogPaths []*datapb.FieldBinlog
for _, binlog := range info.GetBinlogPaths() {
if indexFields.Contain(binlog.GetFieldID()) {
binlogPaths = append(binlogPaths, binlog)
}
}
info.BinlogPaths = binlogPaths
info.Deltalogs = nil
info.Statslogs = nil
return info
})
requestResourceResult, err := loader.requestResource(ctx, indexInfo...)
if err != nil {
return err
}
defer loader.freeRequestResource(requestResourceResult)
mlog.Info(context.TODO(), "segment loader start to load index", mlog.Int("segmentNumAfterFilter", len(infos)))
metrics.QueryNodeLoadSegmentConcurrency.WithLabelValues(paramtable.GetStringNodeID(), "LoadIndex").Inc()
defer metrics.QueryNodeLoadSegmentConcurrency.WithLabelValues(paramtable.GetStringNodeID(), "LoadIndex").Dec()
tr := timerecord.NewTimeRecorder("segmentLoader.LoadIndex")
defer metrics.QueryNodeLoadIndexLatency.WithLabelValues(paramtable.GetStringNodeID()).Observe(float64(tr.ElapseSpan().Milliseconds()))
for _, loadInfo := range infos {
for _, info := range loadInfo.GetIndexInfos() {
if len(info.GetIndexFilePaths()) == 0 {
mlog.Warn(context.TODO(), "failed to add index for segment, index file list is empty, the segment may be too small")
return merr.WrapErrIndexNotFound("index file list empty")
}
err := loader.loadFieldIndex(ctx, segment, info)
if err != nil {
mlog.Warn(context.TODO(), "failed to load index for segment", mlog.Err(err))
return err
}
}
loader.notifyLoadFinish(loadInfo)
}
return loader.waitSegmentLoadDone(ctx, commonpb.SegmentState_SegmentStateNone, []int64{loadInfo.GetSegmentID()}, version)
}
func (loader *segmentLoader) ReopenSegments(ctx context.Context,
loadInfos []*querypb.SegmentLoadInfo,
) error {
@@ -35,7 +35,6 @@ import (
"github.com/milvus-io/milvus/internal/mocks/util/mock_segcore"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/storagev2/packed"
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
"github.com/milvus-io/milvus/internal/util/initcore"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/mlog"
@@ -1397,69 +1396,6 @@ func (suite *SegmentLoaderSuite) TestLoadDeltaLogsChildManifestReadError() {
suite.Contains(err.Error(), "manifest read failed")
}
func (suite *SegmentLoaderSuite) TestLoadIndex() {
ctx := context.Background()
loadInfo := &querypb.SegmentLoadInfo{
SegmentID: 1,
PartitionID: suite.partitionID,
CollectionID: suite.collectionID,
IndexInfos: []*querypb.FieldIndexInfo{
{
IndexFilePaths: []string{},
},
},
InsertChannel: fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", suite.collectionID),
}
segment := &LocalSegment{
baseSegment: baseSegment{loadInfo: atomic.NewPointer[querypb.SegmentLoadInfo](loadInfo)},
bm25StatsHolder: newBM25StatsHolder(),
}
err := suite.loader.LoadIndex(ctx, segment, loadInfo, 0)
suite.ErrorIs(err, merr.ErrIndexNotFound)
}
func (suite *SegmentLoaderSuite) TestLoadIndexWithLimitedResource() {
ctx := context.Background()
loadInfo := &querypb.SegmentLoadInfo{
SegmentID: 1,
PartitionID: suite.partitionID,
CollectionID: suite.collectionID,
IndexInfos: []*querypb.FieldIndexInfo{
{
FieldID: 1,
IndexFilePaths: []string{},
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.IndexINVERTED,
},
},
},
},
BinlogPaths: []*datapb.FieldBinlog{
{
FieldID: 1,
Binlogs: []*datapb.Binlog{
{
LogSize: 1000000000,
MemorySize: 1000000000,
},
},
},
},
}
segment := &LocalSegment{
baseSegment: baseSegment{loadInfo: atomic.NewPointer[querypb.SegmentLoadInfo](loadInfo)},
bm25StatsHolder: newBM25StatsHolder(),
}
paramtable.Get().Save(paramtable.Get().QueryNodeCfg.DiskCapacityLimit.Key, "100000")
defer paramtable.Get().Reset(paramtable.Get().QueryNodeCfg.DiskCapacityLimit.Key)
err := suite.loader.LoadIndex(ctx, segment, loadInfo, 0)
suite.Error(err)
}
func (suite *SegmentLoaderSuite) TestLoadWithMmap() {
key := paramtable.Get().QueryNodeCfg.MmapDirPath.Key
paramtable.Get().Save(key, "/tmp/mmap-test")
+17 -2
View File
@@ -66,6 +66,12 @@ import (
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
// legacyLoadScopeIndex is the retired wire value emitted by QueryCoord
// versions that predate LoadScope_Reopen. Keep the value out of the proto enum
// so it cannot be reused, while recognizing it here to return a precise
// mixed-version protocol error.
const legacyLoadScopeIndex = querypb.LoadScope(2)
type segmentDetacher interface {
DetachStreaming(ctx context.Context, segmentID typeutil.UniqueID) int
}
@@ -536,12 +542,21 @@ func (node *QueryNode) LoadSegments(ctx context.Context, req *querypb.LoadSegmen
switch req.GetLoadScope() {
case querypb.LoadScope_Delta:
return node.loadDeltaLogs(ctx, req), nil
case querypb.LoadScope_Index:
return node.loadIndex(ctx, req), nil
case querypb.LoadScope_Stats:
return node.reopenSegments(ctx, req), nil
case querypb.LoadScope_Reopen:
return node.reopenSegments(ctx, req), nil
case querypb.LoadScope_Full:
// Continue with the full segment load below.
case legacyLoadScopeIndex:
err := merr.WrapErrServiceInternalMsg(
"legacy segment index load scope %d is no longer supported; upgrade QueryCoord to use LoadScope_Reopen before routing requests to this QueryNode",
req.GetLoadScope())
return merr.Status(err), nil
default:
err := merr.WrapErrServiceInternalMsg(
"unsupported segment load scope %d", req.GetLoadScope())
return merr.Status(err), nil
}
// Actual load segment
+26 -117
View File
@@ -840,24 +840,10 @@ func (suite *ServiceSuite) TestLoadDeltaVarchar() {
suite.Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
}
func (suite *ServiceSuite) TestLoadIndex_Success() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
schema := mock_segcore.GenTestCollectionSchema(suite.collectionName, schemapb.DataType_Int64, false)
indexInfos := mock_segcore.GenTestIndexInfoList(suite.collectionID, schema)
infos := suite.genSegmentLoadInfos(schema, indexInfos)
infos = lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
info.SegmentID = info.SegmentID + 1000
return info
})
rawInfo := lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
info = typeutil.Clone(info)
info.IndexInfos = nil
return info
})
func (suite *ServiceSuite) TestLoadSegmentsRejectsUnknownScope() {
ctx := context.Background()
schema := mock_segcore.GenTestCollectionSchema(
suite.collectionName, schemapb.DataType_Int64, false)
req := &querypb.LoadSegmentsRequest{
Base: &commonpb.MsgBase{
MsgID: rand.Int63(),
@@ -865,123 +851,46 @@ func (suite *ServiceSuite) TestLoadIndex_Success() {
},
CollectionID: suite.collectionID,
DstNodeID: suite.node.session.ServerID,
Infos: rawInfo,
Infos: suite.genSegmentLoadInfos(schema, nil),
Schema: schema,
NeedTransfer: false,
LoadScope: querypb.LoadScope_Full,
IndexInfoList: indexInfos,
LoadScope: querypb.LoadScope(99),
IndexInfoList: []*indexpb.IndexInfo{{}},
}
// Load segment
status, err := suite.node.LoadSegments(ctx, req)
suite.Require().NoError(err)
suite.Require().Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
suite.NoError(err)
suite.NotEqual(commonpb.ErrorCode_Success, status.GetErrorCode())
suite.Contains(status.GetReason(), "unsupported segment load scope")
}
for _, segmentID := range lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) int64 {
return info.GetSegmentID()
}) {
suite.Equal(0, len(suite.node.manager.Segment.Get(segmentID).Indexes()))
}
req = &querypb.LoadSegmentsRequest{
func (suite *ServiceSuite) TestLoadSegmentsRejectsLegacyIndexScope() {
ctx := context.Background()
schema := mock_segcore.GenTestCollectionSchema(
suite.collectionName, schemapb.DataType_Int64, false)
req := &querypb.LoadSegmentsRequest{
Base: &commonpb.MsgBase{
MsgID: rand.Int63(),
TargetID: suite.node.session.ServerID,
},
CollectionID: suite.collectionID,
DstNodeID: suite.node.session.ServerID,
Infos: infos,
Infos: suite.genSegmentLoadInfos(schema, nil),
Schema: schema,
NeedTransfer: false,
LoadScope: querypb.LoadScope_Index,
LoadScope: legacyLoadScopeIndex,
IndexInfoList: []*indexpb.IndexInfo{{}},
}
// Load segment
status, err = suite.node.LoadSegments(ctx, req)
suite.Require().NoError(err)
suite.Require().Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
loader := suite.node.loader
suite.node.loader = segments.NewMockLoader(suite.T())
defer func() { suite.node.loader = loader }()
for _, segmentID := range lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) int64 {
return info.GetSegmentID()
}) {
suite.T().Log(segmentID)
suite.T().Log(len(suite.node.manager.Segment.Get(segmentID).Indexes()))
suite.Greater(len(suite.node.manager.Segment.Get(segmentID).Indexes()), 0)
}
}
func (suite *ServiceSuite) TestLoadIndex_Failed() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
schema := mock_segcore.GenTestCollectionSchema(suite.collectionName, schemapb.DataType_Int64, false)
suite.Run("load_non_exist_segment", func() {
indexInfos := mock_segcore.GenTestIndexInfoList(suite.collectionID, schema)
infos := suite.genSegmentLoadInfos(schema, indexInfos)
infos = lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
info.SegmentID = info.SegmentID + 1000
return info
})
rawInfo := lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
info = typeutil.Clone(info)
info.IndexInfos = nil
return info
})
req := &querypb.LoadSegmentsRequest{
Base: &commonpb.MsgBase{
MsgID: rand.Int63(),
TargetID: suite.node.session.ServerID,
},
CollectionID: suite.collectionID,
DstNodeID: suite.node.session.ServerID,
Infos: rawInfo,
Schema: schema,
NeedTransfer: false,
LoadScope: querypb.LoadScope_Index,
IndexInfoList: indexInfos,
}
// Load segment
status, err := suite.node.LoadSegments(ctx, req)
suite.Require().NoError(err)
// Ignore segment missing
suite.Require().Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
})
suite.Run("loader_returns_error", func() {
suite.TestLoadSegments_Int64()
loader := suite.node.loader
mockLoader := segments.NewMockLoader(suite.T())
suite.node.loader = mockLoader
defer func() {
suite.node.loader = loader
}()
mockLoader.EXPECT().LoadIndex(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("mocked error"))
indexInfos := mock_segcore.GenTestIndexInfoList(suite.collectionID, schema)
infos := suite.genSegmentLoadInfos(schema, indexInfos)
req := &querypb.LoadSegmentsRequest{
Base: &commonpb.MsgBase{
MsgID: rand.Int63(),
TargetID: suite.node.session.ServerID,
},
CollectionID: suite.collectionID,
DstNodeID: suite.node.session.ServerID,
Infos: infos,
Schema: schema,
NeedTransfer: false,
LoadScope: querypb.LoadScope_Index,
IndexInfoList: indexInfos,
}
// Load segment
status, err := suite.node.LoadSegments(ctx, req)
suite.Require().NoError(err)
suite.Require().NotEqual(commonpb.ErrorCode_Success, status.GetErrorCode())
})
status, err := suite.node.LoadSegments(ctx, req)
suite.NoError(err)
suite.ErrorIs(merr.Error(status), merr.ErrServiceInternal)
suite.Contains(status.GetReason(), "legacy segment index load scope 2")
suite.Contains(status.GetReason(), "use LoadScope_Reopen")
}
func (suite *ServiceSuite) TestLoadSegments_Failed() {
+1 -1
View File
@@ -447,9 +447,9 @@ message JsonStatsInfo {
}
enum LoadScope {
reserved 2;
Full = 0;
Delta = 1;
Index = 2;
Stats = 3;
Reopen = 4;
}
File diff suppressed because it is too large Load Diff