mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
Cherry pick from master pr: #51323 Related to #51068 - manage JSON index load, replace, and drop through reopen COW snapshots - remove LoadScope_Index and the legacy QueryCoord-to-segcore update path - drop JSON indexes by field and nested path to preserve sibling indexes - regenerate query proto and QueryNode mocks --------- Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
@@ -52,6 +52,30 @@ This design does **not** require the following:
|
||||
|
||||
The goal is **one unified direction with phased delivery**, not a one-shot rewrite of the entire sealed segment implementation.
|
||||
|
||||
### 1.4 Rolling Upgrade Compatibility
|
||||
|
||||
The index-update migration assumes a coordinator baseline whose IndexChecker
|
||||
schedules segment index changes through `LoadScope_Reopen`. The historical
|
||||
`LoadScope_Index` wire value (`2`) is retired and remains reserved so it cannot
|
||||
be reused for another meaning.
|
||||
|
||||
This design does not support a mixed-version deployment in which an older
|
||||
QueryCoord still emits `LoadScope_Index` requests to a newer QueryNode after the
|
||||
legacy index-update handler has been removed. A QueryNode that receives wire
|
||||
value `2` rejects it as an internal protocol mismatch with an explicit upgrade
|
||||
message. The deployment must upgrade QueryCoord to a version that emits
|
||||
`LoadScope_Reopen` before routing index-update requests to these QueryNodes.
|
||||
|
||||
The retired value must not fall through to the full segment load path: an index
|
||||
update is not a full load and must not trigger full-load distribution, bloom
|
||||
filter, or delete-stream side effects. It must also not be blindly remapped to
|
||||
`LoadScope_Reopen`, because a legacy request may omit newer
|
||||
`SegmentLoadInfo` fields; publishing that request as the new load-info snapshot
|
||||
could discard metadata already held by the newer QueryNode. Supporting that
|
||||
version skew would require a dedicated compatibility adapter that merges and
|
||||
normalizes the legacy request against the current published load info before
|
||||
reopen, which is outside the scope of this design.
|
||||
|
||||
---
|
||||
|
||||
## 2. Core Problem
|
||||
|
||||
@@ -263,7 +263,7 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) {
|
||||
auto cload_index_info =
|
||||
static_cast<CLoadIndexInfo>(&load_index_info);
|
||||
AppendIndexV2(trace, cload_index_info);
|
||||
UpdateSealedSegmentIndex(segment.get(), cload_index_info);
|
||||
segment->LoadIndex(load_index_info);
|
||||
};
|
||||
|
||||
// Load pk field
|
||||
|
||||
@@ -284,7 +284,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
|
||||
trace.traceFlags = 0;
|
||||
auto cload_index_info = static_cast<CLoadIndexInfo>(&load_index_info);
|
||||
AppendIndexV2(trace, cload_index_info);
|
||||
UpdateSealedSegmentIndex(segment.get(), cload_index_info);
|
||||
segment->LoadIndex(load_index_info);
|
||||
|
||||
auto unary_range_expr = test::GenUnaryRangeExpr(op_type, literal);
|
||||
auto column_info = test::GenColumnInfo(
|
||||
@@ -562,7 +562,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
|
||||
trace.traceFlags = 0;
|
||||
auto cload_index_info = static_cast<CLoadIndexInfo>(&load_index_info);
|
||||
AppendIndexV2(trace, cload_index_info);
|
||||
UpdateSealedSegmentIndex(segment.get(), cload_index_info);
|
||||
segment->LoadIndex(load_index_info);
|
||||
|
||||
// Test: TermFilterExpr (IN operator)
|
||||
{
|
||||
|
||||
@@ -290,32 +290,6 @@ cancel_and_clear_ngram_indexings(
|
||||
ngram_indexings.clear();
|
||||
}
|
||||
|
||||
template <typename JsonIndexT>
|
||||
static void
|
||||
cancel_and_erase_json_indices(std::vector<JsonIndexT>& json_indices,
|
||||
FieldId field_id,
|
||||
std::string_view nested_path) {
|
||||
auto new_end = std::remove_if(
|
||||
json_indices.begin(), json_indices.end(), [&](auto& index) {
|
||||
auto matched =
|
||||
index.field_id == field_id && index.nested_path == nested_path;
|
||||
if (matched) {
|
||||
cancel_warmup(index.index);
|
||||
}
|
||||
return matched;
|
||||
});
|
||||
json_indices.erase(new_end, json_indices.end());
|
||||
}
|
||||
|
||||
template <typename JsonIndexT>
|
||||
static void
|
||||
cancel_and_clear_json_indices(std::vector<JsonIndexT>& json_indices) {
|
||||
for (auto& index : json_indices) {
|
||||
cancel_warmup(index.index);
|
||||
}
|
||||
json_indices.clear();
|
||||
}
|
||||
|
||||
PinWrapper<const storagev2translator::TimestampIndexCell*>
|
||||
ChunkedSegmentSealedImpl::PinTimestampIndex(
|
||||
const std::shared_ptr<const RuntimeResourceState>& runtime,
|
||||
@@ -370,6 +344,84 @@ ChunkedSegmentSealedImpl::PinPkIndex(milvus::OpContext* op_ctx) const {
|
||||
return PinWrapper<const storagev2translator::PkIndexCell*>(ca, cell);
|
||||
}
|
||||
|
||||
std::vector<PinWrapper<const index::IndexBase*>>
|
||||
ChunkedSegmentSealedImpl::PinJsonIndex(milvus::OpContext* op_ctx,
|
||||
FieldId field_id,
|
||||
const std::string& path,
|
||||
DataType data_type,
|
||||
bool any_type,
|
||||
bool is_array) const {
|
||||
auto runtime = CaptureRuntimeResourceState();
|
||||
int path_len_diff = std::numeric_limits<int>::max();
|
||||
index::CacheIndexBasePtr best_match = nullptr;
|
||||
std::string_view path_view = path;
|
||||
for (const auto& index : runtime->json_indices) {
|
||||
if (index.field_id != field_id) {
|
||||
continue;
|
||||
}
|
||||
switch (index.cast_type.data_type()) {
|
||||
case JsonCastType::DataType::JSON:
|
||||
if (path_view.length() < index.nested_path.length()) {
|
||||
continue;
|
||||
}
|
||||
if (path_view.substr(0, index.nested_path.length()) ==
|
||||
index.nested_path) {
|
||||
int current_len_diff =
|
||||
path_view.length() - index.nested_path.length();
|
||||
if (current_len_diff < path_len_diff) {
|
||||
path_len_diff = current_len_diff;
|
||||
best_match = index.index;
|
||||
}
|
||||
if (path_len_diff == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
if (index.nested_path != path) {
|
||||
continue;
|
||||
}
|
||||
if (any_type || milvus::index::json::IsDataTypeSupported(
|
||||
index.cast_type, data_type, is_array)) {
|
||||
best_match = index.index;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (best_match == nullptr) {
|
||||
return {};
|
||||
}
|
||||
auto ca = SemiInlineGet(best_match->PinCells(op_ctx, {0}));
|
||||
auto pinned_index = ca->get_cell_of(0);
|
||||
return {PinWrapper<const index::IndexBase*>(std::move(ca), pinned_index)};
|
||||
}
|
||||
|
||||
std::string
|
||||
ChunkedSegmentSealedImpl::GetJsonFlatIndexNestedPath(
|
||||
FieldId field_id, std::string_view query_path) const {
|
||||
auto runtime = CaptureRuntimeResourceState();
|
||||
std::string best_path;
|
||||
int path_len_diff = std::numeric_limits<int>::max();
|
||||
for (const auto& index : runtime->json_indices) {
|
||||
if (index.field_id != field_id ||
|
||||
index.cast_type.data_type() != JsonCastType::DataType::JSON ||
|
||||
query_path.length() < index.nested_path.length() ||
|
||||
query_path.substr(0, index.nested_path.length()) !=
|
||||
index.nested_path) {
|
||||
continue;
|
||||
}
|
||||
int current_len_diff = query_path.length() - index.nested_path.length();
|
||||
if (current_len_diff < path_len_diff) {
|
||||
path_len_diff = current_len_diff;
|
||||
best_path = index.nested_path;
|
||||
}
|
||||
if (path_len_diff == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return best_path;
|
||||
}
|
||||
|
||||
bool
|
||||
ChunkedSegmentSealedImpl::Contain(const PkType& pk) const {
|
||||
auto schema_snapshot = CaptureSchemaSnapshot();
|
||||
@@ -593,8 +645,12 @@ ChunkedSegmentSealedImpl::LoadIndex(LoadIndexInfo& info,
|
||||
LoadVecIndex(
|
||||
info, schema_snapshot, is_replace, staged_state, committer);
|
||||
} else {
|
||||
LoadScalarIndex(
|
||||
info, schema_snapshot, is_replace, runtime, staged_state);
|
||||
LoadScalarIndex(info,
|
||||
schema_snapshot,
|
||||
is_replace,
|
||||
runtime,
|
||||
staged_state,
|
||||
committer);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -705,7 +761,8 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
|
||||
const SchemaPtr& schema_snapshot,
|
||||
bool is_replace,
|
||||
RuntimeResourceState* runtime,
|
||||
PublishedSegmentState* staged_state) {
|
||||
PublishedSegmentState* staged_state,
|
||||
StagedStateCommitter* committer) {
|
||||
// NOTE: lock only when data is ready to avoid starvation
|
||||
auto field_id = FieldId(info.field_id);
|
||||
auto snapshot = CapturePublishedState();
|
||||
@@ -716,6 +773,26 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
|
||||
|
||||
RuntimeResourceState* target_runtime = runtime;
|
||||
std::shared_ptr<RuntimeResourceState> owned_runtime;
|
||||
std::vector<index::CacheIndexBasePtr> retired_indexings;
|
||||
|
||||
auto retire_indexing = [&](index::CacheIndexBasePtr indexing) {
|
||||
if (indexing == nullptr) {
|
||||
return;
|
||||
}
|
||||
if (committer != nullptr) {
|
||||
committer->RetireCacheIndexingLocked(std::move(indexing));
|
||||
} else if (owned_runtime != nullptr) {
|
||||
retired_indexings.push_back(std::move(indexing));
|
||||
}
|
||||
};
|
||||
|
||||
auto cancel_retired_indexings = [&] {
|
||||
for (auto& indexing : retired_indexings) {
|
||||
if (indexing != nullptr) {
|
||||
indexing->CancelWarmup();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
LOG_INFO("LoadScalarIndex, fieldID:{}. segmentID:{}, is_pk:{}",
|
||||
info.field_id,
|
||||
@@ -754,19 +831,19 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
|
||||
|
||||
if (field_meta.get_data_type() == DataType::JSON) {
|
||||
auto path = info.index_params.at(JSON_PATH);
|
||||
if (target_runtime == nullptr) {
|
||||
owned_runtime = CloneMutableRuntimeResourceState();
|
||||
target_runtime = owned_runtime.get();
|
||||
}
|
||||
for (auto& retired :
|
||||
EraseJsonIndexesAtPath(*target_runtime, field_id, path)) {
|
||||
retire_indexing(std::move(retired));
|
||||
}
|
||||
if (auto it = info.index_params.find(index::INDEX_TYPE);
|
||||
it != info.index_params.end() &&
|
||||
it->second == index::NGRAM_INDEX_TYPE) {
|
||||
if (target_runtime == nullptr) {
|
||||
owned_runtime = CloneMutableRuntimeResourceState();
|
||||
target_runtime = owned_runtime.get();
|
||||
}
|
||||
auto& path_indexings = target_runtime->ngram_indexings[field_id];
|
||||
if (auto path_it = path_indexings.find(path);
|
||||
path_it != path_indexings.end()) {
|
||||
cancel_warmup(path_it->second);
|
||||
}
|
||||
path_indexings[path] = std::move(info.cache_index);
|
||||
target_runtime->ngram_indexings[field_id][path] =
|
||||
std::move(info.cache_index);
|
||||
if (staged_state != nullptr) {
|
||||
clear_bit_if_present(
|
||||
staged_state->published_binlog_index_ready_bitset,
|
||||
@@ -781,6 +858,7 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
|
||||
? ToConstRuntimeState(std::move(owned_runtime))
|
||||
: FreezeRuntimeResourceState(*target_runtime);
|
||||
PublishIndexReadyLocked(field_id, false, published_runtime);
|
||||
cancel_retired_indexings();
|
||||
}
|
||||
return;
|
||||
} else {
|
||||
@@ -790,10 +868,21 @@ ChunkedSegmentSealedImpl::LoadScalarIndex(LoadIndexInfo& info,
|
||||
index.index = std::move(info.cache_index);
|
||||
index.cast_type =
|
||||
JsonCastType::FromString(info.index_params.at(JSON_CAST_TYPE));
|
||||
json_indices.withWLock([&](auto& json_indexings) {
|
||||
cancel_and_erase_json_indices(json_indexings, field_id, path);
|
||||
json_indexings.push_back(std::move(index));
|
||||
});
|
||||
target_runtime->json_indices.push_back(std::move(index));
|
||||
if (staged_state != nullptr) {
|
||||
SyncJsonNgramIndexState(
|
||||
*staged_state, *target_runtime, field_id);
|
||||
NormalizePublishedState(*staged_state);
|
||||
} else if (owned_runtime != nullptr) {
|
||||
auto published_runtime =
|
||||
ToConstRuntimeState(std::move(owned_runtime));
|
||||
MutatePublishedStateLocked([&](PublishedSegmentState& state) {
|
||||
state.runtime = published_runtime;
|
||||
SyncJsonNgramIndexState(
|
||||
state, *published_runtime, field_id);
|
||||
});
|
||||
cancel_retired_indexings();
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -904,6 +993,7 @@ ChunkedSegmentSealedImpl::CloneRuntimeResourceState(
|
||||
state->ngram_indexings = current->ngram_indexings;
|
||||
state->text_lob_paths = current->text_lob_paths;
|
||||
state->text_indexes = current->text_indexes;
|
||||
state->json_indices = current->json_indices;
|
||||
state->json_stats = current->json_stats;
|
||||
state->reader = current->reader;
|
||||
state->timestamps = current->timestamps;
|
||||
@@ -1013,6 +1103,87 @@ ChunkedSegmentSealedImpl::DropVectorIndexing(RuntimeResourceState& runtime,
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<index::CacheIndexBasePtr>
|
||||
ChunkedSegmentSealedImpl::EraseJsonIndexings(RuntimeResourceState& runtime,
|
||||
FieldId field_id,
|
||||
std::string_view nested_path) {
|
||||
std::vector<index::CacheIndexBasePtr> retired;
|
||||
auto new_end = std::remove_if(runtime.json_indices.begin(),
|
||||
runtime.json_indices.end(),
|
||||
[&](JsonIndex& index) {
|
||||
if (index.field_id != field_id ||
|
||||
index.nested_path != nested_path) {
|
||||
return false;
|
||||
}
|
||||
retired.push_back(std::move(index.index));
|
||||
return true;
|
||||
});
|
||||
runtime.json_indices.erase(new_end, runtime.json_indices.end());
|
||||
return retired;
|
||||
}
|
||||
|
||||
index::CacheIndexBasePtr
|
||||
ChunkedSegmentSealedImpl::EraseJsonNgramIndexing(RuntimeResourceState& runtime,
|
||||
FieldId field_id,
|
||||
std::string_view nested_path) {
|
||||
auto field_it = runtime.ngram_indexings.find(field_id);
|
||||
if (field_it == runtime.ngram_indexings.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto& path_indexings = field_it->second;
|
||||
auto path_it = path_indexings.find(std::string(nested_path));
|
||||
if (path_it == path_indexings.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto retired = std::move(path_it->second);
|
||||
path_indexings.erase(path_it);
|
||||
if (path_indexings.empty()) {
|
||||
runtime.ngram_indexings.erase(field_it);
|
||||
}
|
||||
return retired;
|
||||
}
|
||||
|
||||
std::vector<index::CacheIndexBasePtr>
|
||||
ChunkedSegmentSealedImpl::EraseJsonIndexesAtPath(RuntimeResourceState& runtime,
|
||||
FieldId field_id,
|
||||
std::string_view nested_path) {
|
||||
auto retired = EraseJsonIndexings(runtime, field_id, nested_path);
|
||||
if (auto ngram = EraseJsonNgramIndexing(runtime, field_id, nested_path);
|
||||
ngram != nullptr) {
|
||||
retired.push_back(std::move(ngram));
|
||||
}
|
||||
return retired;
|
||||
}
|
||||
|
||||
bool
|
||||
ChunkedSegmentSealedImpl::RuntimeJsonNgramIndexReady(
|
||||
const RuntimeResourceState& runtime, FieldId field_id) {
|
||||
auto it = runtime.ngram_indexings.find(field_id);
|
||||
return it != runtime.ngram_indexings.end() && !it->second.empty();
|
||||
}
|
||||
|
||||
void
|
||||
ChunkedSegmentSealedImpl::SyncJsonNgramIndexState(
|
||||
PublishedSegmentState& state,
|
||||
const RuntimeResourceState& runtime,
|
||||
FieldId field_id) {
|
||||
if (RuntimeJsonNgramIndexReady(runtime, field_id)) {
|
||||
set_bit(state.published_index_ready_bitset, field_id, true);
|
||||
SetPublishedIndexRawDataInState(state, field_id, false);
|
||||
return;
|
||||
}
|
||||
|
||||
clear_bit_if_present(state.published_index_ready_bitset, field_id);
|
||||
clear_bit_if_present(state.index_ready_bitset, field_id);
|
||||
if (!get_bit_if_present(state.published_binlog_index_ready_bitset,
|
||||
field_id)) {
|
||||
ClearPublishedIndexRawDataInState(state, field_id);
|
||||
ClearIndexRawDataInState(state, field_id);
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<ChunkedSegmentSealedImpl::PublishedSegmentState>
|
||||
ChunkedSegmentSealedImpl::ClonePublishedState(
|
||||
const std::shared_ptr<const PublishedSegmentState>& current) const {
|
||||
@@ -1114,6 +1285,16 @@ ChunkedSegmentSealedImpl::NormalizePublishedState(
|
||||
state.index_has_raw_data[field_id] = raw_it->second;
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& [field_id, path_indexings] :
|
||||
state.runtime->ngram_indexings) {
|
||||
if (path_indexings.empty() ||
|
||||
!field_exists_in_schema(state.schema, field_id)) {
|
||||
continue;
|
||||
}
|
||||
set_bit(state.index_ready_bitset, field_id, true);
|
||||
state.index_has_raw_data[field_id] = false;
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < state.published_index_ready_bitset.size(); ++i) {
|
||||
@@ -1206,6 +1387,7 @@ ChunkedSegmentSealedImpl::FreezeRuntimeResourceState(
|
||||
runtime->ngram_indexings = current.ngram_indexings;
|
||||
runtime->text_lob_paths = current.text_lob_paths;
|
||||
runtime->text_indexes = current.text_indexes;
|
||||
runtime->json_indices = current.json_indices;
|
||||
runtime->json_stats = current.json_stats;
|
||||
runtime->reader = current.reader;
|
||||
runtime->timestamps = current.timestamps;
|
||||
@@ -3682,28 +3864,19 @@ void
|
||||
ChunkedSegmentSealedImpl::DropJSONIndex(const FieldId field_id,
|
||||
const std::string& nested_path) {
|
||||
std::lock_guard<std::mutex> reopen_guard(reopen_mutex_);
|
||||
{
|
||||
std::unique_lock lck(mutex_);
|
||||
json_indices.withWLock([&](auto& json_indexings) {
|
||||
cancel_and_erase_json_indices(
|
||||
json_indexings, field_id, nested_path);
|
||||
});
|
||||
}
|
||||
|
||||
auto next_runtime = CloneMutableRuntimeResourceState();
|
||||
auto field_it = next_runtime->ngram_indexings.find(field_id);
|
||||
if (field_it != next_runtime->ngram_indexings.end()) {
|
||||
auto& path_indexings = field_it->second;
|
||||
if (auto path_it = path_indexings.find(nested_path);
|
||||
path_it != path_indexings.end()) {
|
||||
cancel_warmup(path_it->second);
|
||||
path_indexings.erase(path_it);
|
||||
}
|
||||
if (path_indexings.empty()) {
|
||||
next_runtime->ngram_indexings.erase(field_it);
|
||||
auto retired_indexings =
|
||||
EraseJsonIndexesAtPath(*next_runtime, field_id, nested_path);
|
||||
auto published_runtime = ToConstRuntimeState(std::move(next_runtime));
|
||||
MutatePublishedStateLocked([&](PublishedSegmentState& state) {
|
||||
state.runtime = published_runtime;
|
||||
SyncJsonNgramIndexState(state, *published_runtime, field_id);
|
||||
});
|
||||
for (auto& indexing : retired_indexings) {
|
||||
if (indexing != nullptr) {
|
||||
indexing->CancelWarmup();
|
||||
}
|
||||
}
|
||||
PublishRuntimeStateLocked(ToConstRuntimeState(std::move(next_runtime)));
|
||||
}
|
||||
|
||||
void
|
||||
@@ -4763,6 +4936,15 @@ ChunkedSegmentSealedImpl::ClearData() {
|
||||
entry->indexing_->CancelWarmup();
|
||||
}
|
||||
}
|
||||
for (const auto& json_index : runtime_snapshot->json_indices) {
|
||||
cancel_warmup(json_index.index);
|
||||
}
|
||||
for (const auto& [_, path_indexings] :
|
||||
runtime_snapshot->ngram_indexings) {
|
||||
for (const auto& [__, indexing] : path_indexings) {
|
||||
cancel_warmup(indexing);
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
std::unique_lock lck(mutex_);
|
||||
@@ -4774,9 +4956,6 @@ ChunkedSegmentSealedImpl::ClearData() {
|
||||
ngram_indexings_.withWLock([&](auto& ngram_indexings) {
|
||||
cancel_and_clear_ngram_indexings(ngram_indexings);
|
||||
});
|
||||
json_indices.withWLock([&](auto& json_indexings) {
|
||||
cancel_and_clear_json_indices(json_indexings);
|
||||
});
|
||||
insert_record_.clear();
|
||||
timestamp_index_slot_.wlock()->reset();
|
||||
pk_index_slot_.wlock()->reset();
|
||||
@@ -5821,18 +6000,16 @@ ChunkedSegmentSealedImpl::HasIndex(FieldId field_id) const {
|
||||
|
||||
bool
|
||||
ChunkedSegmentSealedImpl::HasJsonIndex(FieldId field_id) const {
|
||||
// JSON indexes (JsonFlatIndex + JSON-cast) live in a separate per-segment
|
||||
// vector rather than in either index bitset. Kept as a distinct API so
|
||||
// HasIndex() preserves its narrower "scalar/vector/binlog index exists"
|
||||
// semantics for ReorderConjunctExpr and other consumers.
|
||||
return json_indices.withRLock([&](const auto& vec) {
|
||||
for (const auto& index : vec) {
|
||||
if (index.field_id == field_id) {
|
||||
return true;
|
||||
}
|
||||
// JSON indexes (JsonFlatIndex + JSON-cast) remain distinct from the
|
||||
// scalar/vector/binlog readiness bitsets, but their ownership now follows
|
||||
// the published runtime snapshot.
|
||||
auto runtime = CaptureRuntimeResourceState();
|
||||
for (const auto& index : runtime->json_indices) {
|
||||
if (index.field_id == field_id) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
@@ -6807,6 +6984,21 @@ ChunkedSegmentSealedImpl::FinalizeLoadDiffForReopen(
|
||||
}
|
||||
}
|
||||
|
||||
CheckCancellation(op_ctx, id_, "ChunkedSegmentSealedImpl::ApplyLoadDiff()");
|
||||
for (const auto& [field_id, nested_paths] : diff.json_indexes_to_drop) {
|
||||
for (const auto& nested_path : nested_paths) {
|
||||
committer.Commit([&, field_id, nested_path](
|
||||
RuntimeResourceState& runtime,
|
||||
PublishedSegmentState& staged_state) {
|
||||
for (auto& retired :
|
||||
EraseJsonIndexesAtPath(runtime, field_id, nested_path)) {
|
||||
committer.RetireCacheIndexingLocked(std::move(retired));
|
||||
}
|
||||
SyncJsonNgramIndexState(staged_state, runtime, field_id);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
CheckCancellation(op_ctx, id_, "ChunkedSegmentSealedImpl::ApplyLoadDiff()");
|
||||
if (!diff.json_stats_to_drop.empty()) {
|
||||
for (auto field_id : diff.json_stats_to_drop) {
|
||||
|
||||
@@ -184,6 +184,18 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
return {PinWrapper<const index::IndexBase*>(std::move(ca), index)};
|
||||
}
|
||||
|
||||
std::vector<PinWrapper<const index::IndexBase*>>
|
||||
PinJsonIndex(milvus::OpContext* op_ctx,
|
||||
FieldId field_id,
|
||||
const std::string& path,
|
||||
DataType data_type,
|
||||
bool any_type,
|
||||
bool is_array) const override;
|
||||
|
||||
std::string
|
||||
GetJsonFlatIndexNestedPath(FieldId field_id,
|
||||
std::string_view query_path) const override;
|
||||
|
||||
bool
|
||||
Contain(const PkType& pk) const override;
|
||||
|
||||
@@ -322,6 +334,13 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
std::shared_ptr<milvus::cachinglayer::CacheSlot<
|
||||
milvus::index::TextMatchIndex>>>;
|
||||
|
||||
struct JsonIndex {
|
||||
FieldId field_id;
|
||||
std::string nested_path;
|
||||
JsonCastType cast_type{JsonCastType::UNKNOWN};
|
||||
index::CacheIndexBasePtr index;
|
||||
};
|
||||
|
||||
// When non-zero commit_ts is active, every row in this segment carries it
|
||||
// as its effective row timestamp (load-time overwrite). All timestamp
|
||||
// consumers must route through this so the override applies uniformly on
|
||||
@@ -347,6 +366,7 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
ngram_indexings;
|
||||
std::unordered_map<FieldId, std::string> text_lob_paths;
|
||||
std::unordered_map<FieldId, TextIndexVariant> text_indexes;
|
||||
std::vector<JsonIndex> json_indices;
|
||||
std::unordered_map<FieldId, std::shared_ptr<index::JsonKeyStats>>
|
||||
json_stats;
|
||||
std::shared_ptr<milvus_storage::api::Reader> reader;
|
||||
@@ -1314,7 +1334,8 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
const SchemaPtr& schema_snapshot,
|
||||
bool is_replace = false,
|
||||
RuntimeResourceState* runtime = nullptr,
|
||||
PublishedSegmentState* staged_state = nullptr);
|
||||
PublishedSegmentState* staged_state = nullptr,
|
||||
StagedStateCommitter* committer = nullptr);
|
||||
|
||||
void
|
||||
LoadIndex(LoadIndexInfo& info, bool is_replace);
|
||||
@@ -1408,6 +1429,30 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
static void
|
||||
DropVectorIndexing(RuntimeResourceState& runtime, FieldId field_id);
|
||||
|
||||
static std::vector<index::CacheIndexBasePtr>
|
||||
EraseJsonIndexings(RuntimeResourceState& runtime,
|
||||
FieldId field_id,
|
||||
std::string_view nested_path);
|
||||
|
||||
static index::CacheIndexBasePtr
|
||||
EraseJsonNgramIndexing(RuntimeResourceState& runtime,
|
||||
FieldId field_id,
|
||||
std::string_view nested_path);
|
||||
|
||||
static std::vector<index::CacheIndexBasePtr>
|
||||
EraseJsonIndexesAtPath(RuntimeResourceState& runtime,
|
||||
FieldId field_id,
|
||||
std::string_view nested_path);
|
||||
|
||||
static bool
|
||||
RuntimeJsonNgramIndexReady(const RuntimeResourceState& runtime,
|
||||
FieldId field_id);
|
||||
|
||||
static void
|
||||
SyncJsonNgramIndexState(PublishedSegmentState& state,
|
||||
const RuntimeResourceState& runtime,
|
||||
FieldId field_id);
|
||||
|
||||
std::shared_ptr<PublishedSegmentState>
|
||||
BuildNextPublishedState(
|
||||
const std::shared_ptr<const PublishedSegmentState>& current,
|
||||
@@ -1489,17 +1534,31 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
Publish(const std::shared_ptr<const PublishedSegmentState>& current,
|
||||
const StateDelta& delta) {
|
||||
std::vector<SealedIndexingEntryPtr> retired_indexings;
|
||||
std::vector<index::CacheIndexBasePtr> retired_cache_indexings;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
segment_.PublishState(
|
||||
segment_.BuildNextPublishedState(current, delta));
|
||||
retired_indexings.swap(retired_vector_indexings_);
|
||||
retired_cache_indexings.swap(retired_cache_indexings_);
|
||||
}
|
||||
for (auto& entry : retired_indexings) {
|
||||
if (entry != nullptr && entry->indexing_ != nullptr) {
|
||||
entry->indexing_->CancelWarmup();
|
||||
}
|
||||
}
|
||||
for (auto& indexing : retired_cache_indexings) {
|
||||
if (indexing != nullptr) {
|
||||
indexing->CancelWarmup();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
RetireCacheIndexingLocked(index::CacheIndexBasePtr indexing) {
|
||||
if (indexing != nullptr) {
|
||||
retired_cache_indexings_.push_back(std::move(indexing));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
@@ -1515,6 +1574,7 @@ class ChunkedSegmentSealedImpl : public SegmentSealed {
|
||||
RuntimeResourceState* runtime_;
|
||||
PublishedSegmentState* staged_state_;
|
||||
std::vector<SealedIndexingEntryPtr> retired_vector_indexings_;
|
||||
std::vector<index::CacheIndexBasePtr> retired_cache_indexings_;
|
||||
std::mutex mutex_;
|
||||
};
|
||||
|
||||
|
||||
@@ -291,6 +291,17 @@ SegmentLoadInfo::ComputeDiffIndexes(LoadDiff& diff, SegmentLoadInfo& new_info) {
|
||||
new_index_ids.insert(index_id);
|
||||
}
|
||||
}
|
||||
std::unordered_map<FieldId, std::unordered_set<std::string>>
|
||||
new_json_index_paths;
|
||||
for (const auto& [field_id, index_paths] :
|
||||
new_info.json_index_path_cache_) {
|
||||
if (!new_info.HasFieldInSchema(field_id)) {
|
||||
continue;
|
||||
}
|
||||
for (const auto& index_path : index_paths) {
|
||||
new_json_index_paths[field_id].insert(index_path.second);
|
||||
}
|
||||
}
|
||||
// Find indexes to load/replace: indexes in new_info but not in current
|
||||
// Only consider fields that exist in the current schema (skip dropped fields)
|
||||
for (const auto& [field_id, load_index_infos] :
|
||||
@@ -321,6 +332,18 @@ SegmentLoadInfo::ComputeDiffIndexes(LoadDiff& diff, SegmentLoadInfo& new_info) {
|
||||
for (auto index_id : index_ids) {
|
||||
if (!new_info.HasFieldInSchema(field_id) ||
|
||||
new_index_ids.find(index_id) == new_index_ids.end()) {
|
||||
auto field_paths = json_index_path_cache_.find(field_id);
|
||||
if (field_paths != json_index_path_cache_.end()) {
|
||||
auto path = field_paths->second.find(index_id);
|
||||
if (path != field_paths->second.end()) {
|
||||
if (new_json_index_paths[field_id].find(path->second) ==
|
||||
new_json_index_paths[field_id].end()) {
|
||||
diff.json_indexes_to_drop[field_id].insert(
|
||||
path->second);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
diff.indexes_to_drop.insert(field_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +86,12 @@ struct LoadDiff {
|
||||
// Indexes that need to be dropped (field_id set)
|
||||
std::set<FieldId> indexes_to_drop;
|
||||
|
||||
// JSON indexes that need to be dropped, keyed by field and nested path.
|
||||
// JSON fields may have multiple indexes, so a field-only drop loses the
|
||||
// identity needed to preserve sibling paths.
|
||||
std::unordered_map<FieldId, std::unordered_set<std::string>>
|
||||
json_indexes_to_drop;
|
||||
|
||||
// Field data that need to be dropped (field_id set)
|
||||
// Only populated when both current and new use binlog mode
|
||||
std::unordered_set<FieldId> field_data_to_drop;
|
||||
@@ -136,7 +142,8 @@ struct LoadDiff {
|
||||
!column_groups_to_lazyload.empty() ||
|
||||
!column_groups_to_lazyreplace.empty() ||
|
||||
!fields_to_reload.empty() || !indexes_to_drop.empty() ||
|
||||
!field_data_to_drop.empty() || !fields_to_fill_default.empty() ||
|
||||
!json_indexes_to_drop.empty() || !field_data_to_drop.empty() ||
|
||||
!fields_to_fill_default.empty() ||
|
||||
!text_indexes_to_load.empty() || !json_stats_to_load.empty() ||
|
||||
!json_stats_to_replace.empty() || !json_stats_to_drop.empty() ||
|
||||
!text_indexes_to_create.empty() || manifest_updated ||
|
||||
@@ -289,6 +296,19 @@ struct LoadDiff {
|
||||
}
|
||||
oss << "], ";
|
||||
|
||||
// json_indexes_to_drop
|
||||
oss << "json_indexes_to_drop=[";
|
||||
first = true;
|
||||
for (const auto& [field_id, paths] : json_indexes_to_drop) {
|
||||
for (const auto& path : paths) {
|
||||
if (!first)
|
||||
oss << ", ";
|
||||
first = false;
|
||||
oss << field_id.get() << ":" << path;
|
||||
}
|
||||
}
|
||||
oss << "], ";
|
||||
|
||||
// field_data_to_drop
|
||||
oss << "field_data_to_drop=[";
|
||||
first = true;
|
||||
@@ -424,6 +444,7 @@ class SegmentLoadInfo {
|
||||
schema_(other.schema_),
|
||||
converted_field_index_cache_(other.converted_field_index_cache_),
|
||||
field_index_id_cache_(other.field_index_id_cache_),
|
||||
json_index_path_cache_(other.json_index_path_cache_),
|
||||
field_index_has_raw_data_(other.field_index_has_raw_data_),
|
||||
fields_filled_with_default_(other.fields_filled_with_default_),
|
||||
column_groups_(other.column_groups_),
|
||||
@@ -440,6 +461,7 @@ class SegmentLoadInfo {
|
||||
converted_field_index_cache_(
|
||||
std::move(other.converted_field_index_cache_)),
|
||||
field_index_id_cache_(std::move(other.field_index_id_cache_)),
|
||||
json_index_path_cache_(std::move(other.json_index_path_cache_)),
|
||||
field_index_has_raw_data_(std::move(other.field_index_has_raw_data_)),
|
||||
fields_filled_with_default_(
|
||||
std::move(other.fields_filled_with_default_)),
|
||||
@@ -460,6 +482,7 @@ class SegmentLoadInfo {
|
||||
schema_ = other.schema_;
|
||||
converted_field_index_cache_ = other.converted_field_index_cache_;
|
||||
field_index_id_cache_ = other.field_index_id_cache_;
|
||||
json_index_path_cache_ = other.json_index_path_cache_;
|
||||
field_index_has_raw_data_ = other.field_index_has_raw_data_;
|
||||
column_groups_ = other.column_groups_;
|
||||
fields_filled_with_default_ = other.fields_filled_with_default_;
|
||||
@@ -480,6 +503,7 @@ class SegmentLoadInfo {
|
||||
converted_field_index_cache_ =
|
||||
std::move(other.converted_field_index_cache_);
|
||||
field_index_id_cache_ = std::move(other.field_index_id_cache_);
|
||||
json_index_path_cache_ = std::move(other.json_index_path_cache_);
|
||||
field_index_has_raw_data_ =
|
||||
std::move(other.field_index_has_raw_data_);
|
||||
fields_filled_with_default_ =
|
||||
@@ -1167,6 +1191,7 @@ class SegmentLoadInfo {
|
||||
|
||||
prune_map(converted_field_index_cache_);
|
||||
prune_map(field_index_id_cache_);
|
||||
prune_map(json_index_path_cache_);
|
||||
prune_set(field_index_has_raw_data_);
|
||||
prune_set(fields_filled_with_default_);
|
||||
prune_set(created_text_indexes_);
|
||||
@@ -1190,6 +1215,7 @@ class SegmentLoadInfo {
|
||||
// Convert index infos to LoadIndexInfo and build per-field cache
|
||||
converted_field_index_cache_.clear();
|
||||
field_index_id_cache_.clear();
|
||||
json_index_path_cache_.clear();
|
||||
field_index_has_raw_data_.clear();
|
||||
for (int i = 0; i < info_.index_infos_size(); i++) {
|
||||
const auto& index_info = info_.index_infos(i);
|
||||
@@ -1225,6 +1251,13 @@ class SegmentLoadInfo {
|
||||
if (CheckIndexHasRawData(load_index_info)) {
|
||||
field_index_has_raw_data_.insert(field_id);
|
||||
}
|
||||
if (load_index_info.field_type == DataType::JSON) {
|
||||
auto path_it = load_index_info.index_params.find(JSON_PATH);
|
||||
if (path_it != load_index_info.index_params.end()) {
|
||||
json_index_path_cache_[field_id][index_info.indexid()] =
|
||||
path_it->second;
|
||||
}
|
||||
}
|
||||
converted_field_index_cache_[field_id].push_back(
|
||||
std::move(load_index_info));
|
||||
}
|
||||
@@ -1267,6 +1300,12 @@ class SegmentLoadInfo {
|
||||
// needs to know which index ids are already present per field.
|
||||
std::unordered_map<FieldId, std::vector<int64_t>> field_index_id_cache_;
|
||||
|
||||
// Lightweight JSON index identity retained after manifest load-info
|
||||
// compaction so reopen can drop one nested path without affecting sibling
|
||||
// indexes on the same JSON field.
|
||||
std::unordered_map<FieldId, std::unordered_map<int64_t, std::string>>
|
||||
json_index_path_cache_;
|
||||
|
||||
// set of field ids that corresponding index has raw data
|
||||
std::set<FieldId> field_index_has_raw_data_;
|
||||
|
||||
|
||||
@@ -2323,6 +2323,55 @@ TEST_F(SegmentLoadInfoTest, ComputeDiffIndexReplaceDropConflict) {
|
||||
EXPECT_TRUE(actually_dropped.empty());
|
||||
}
|
||||
|
||||
TEST_F(SegmentLoadInfoTest, ComputeDiffDropsJsonIndexByNestedPath) {
|
||||
auto add_json_index = [](proto::segcore::SegmentLoadInfo& proto,
|
||||
int64_t index_id,
|
||||
const std::string& nested_path) {
|
||||
auto* index = proto.add_index_infos();
|
||||
index->set_fieldid(102);
|
||||
index->set_indexid(index_id);
|
||||
index->add_index_file_paths("/path/to/json_index_" +
|
||||
std::to_string(index_id));
|
||||
auto* type = index->add_index_params();
|
||||
type->set_key(milvus::index::INDEX_TYPE);
|
||||
type->set_value(milvus::index::INVERTED_INDEX_TYPE);
|
||||
auto* path = index->add_index_params();
|
||||
path->set_key(JSON_PATH);
|
||||
path->set_value(nested_path);
|
||||
auto* cast = index->add_index_params();
|
||||
cast->set_key(JSON_CAST_TYPE);
|
||||
cast->set_value("DOUBLE");
|
||||
};
|
||||
|
||||
proto::segcore::SegmentLoadInfo current_proto;
|
||||
current_proto.set_segmentid(100);
|
||||
current_proto.set_num_of_rows(1000);
|
||||
current_proto.set_manifest_path("/path/to/manifest");
|
||||
add_json_index(current_proto, 5001, "a");
|
||||
add_json_index(current_proto, 5002, "b");
|
||||
|
||||
proto::segcore::SegmentLoadInfo new_proto;
|
||||
new_proto.set_segmentid(100);
|
||||
new_proto.set_num_of_rows(1000);
|
||||
new_proto.set_manifest_path("/path/to/manifest");
|
||||
add_json_index(new_proto, 5002, "b");
|
||||
|
||||
SegmentLoadInfo current_info(current_proto, schema_);
|
||||
SegmentLoadInfo new_info(new_proto, schema_);
|
||||
current_info.SetColumnGroupsForTesting(
|
||||
std::make_shared<milvus_storage::api::ColumnGroups>());
|
||||
new_info.SetColumnGroupsForTesting(
|
||||
std::make_shared<milvus_storage::api::ColumnGroups>());
|
||||
current_info.CompactRuntimeInfoForManifest();
|
||||
EXPECT_EQ(current_info.GetIndexInfoCount(), 0);
|
||||
auto diff = current_info.ComputeDiff(new_info);
|
||||
|
||||
ASSERT_EQ(diff.json_indexes_to_drop.count(FieldId(102)), 1);
|
||||
EXPECT_EQ(diff.json_indexes_to_drop.at(FieldId(102)).count("a"), 1);
|
||||
EXPECT_EQ(diff.json_indexes_to_drop.at(FieldId(102)).count("b"), 0);
|
||||
EXPECT_EQ(diff.indexes_to_drop.count(FieldId(102)), 0);
|
||||
}
|
||||
|
||||
TEST_F(SegmentLoadInfoTest, ComputeDiffBinlogReplace) {
|
||||
// When a field's binlog group changes, it goes to binlogs_to_replace
|
||||
|
||||
|
||||
@@ -59,112 +59,6 @@ class SegmentSealed : public SegmentInternalInterface {
|
||||
virtual InsertRecord<true>&
|
||||
get_insert_record() = 0;
|
||||
|
||||
virtual std::vector<PinWrapper<const index::IndexBase*>>
|
||||
PinJsonIndex(milvus::OpContext* op_ctx,
|
||||
FieldId field_id,
|
||||
const std::string& path,
|
||||
DataType data_type,
|
||||
bool any_type,
|
||||
bool is_array) const override {
|
||||
int path_len_diff = std::numeric_limits<int>::max();
|
||||
index::CacheIndexBasePtr best_match = nullptr;
|
||||
std::string_view path_view = path;
|
||||
auto res = json_indices.withRLock(
|
||||
[&](auto vec) -> PinWrapper<const index::IndexBase*> {
|
||||
for (const auto& index : vec) {
|
||||
if (index.field_id != field_id) {
|
||||
continue;
|
||||
}
|
||||
switch (index.cast_type.data_type()) {
|
||||
case JsonCastType::DataType::JSON:
|
||||
if (path_view.length() <
|
||||
index.nested_path.length()) {
|
||||
continue;
|
||||
}
|
||||
if (path_view.substr(0,
|
||||
index.nested_path.length()) ==
|
||||
index.nested_path) {
|
||||
int current_len_diff =
|
||||
path_view.length() -
|
||||
index.nested_path.length();
|
||||
if (current_len_diff < path_len_diff) {
|
||||
path_len_diff = current_len_diff;
|
||||
best_match = index.index;
|
||||
}
|
||||
if (path_len_diff == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
if (index.nested_path != path) {
|
||||
continue;
|
||||
}
|
||||
if (any_type) {
|
||||
best_match = index.index;
|
||||
break;
|
||||
}
|
||||
if (milvus::index::json::IsDataTypeSupported(
|
||||
index.cast_type, data_type, is_array)) {
|
||||
best_match = index.index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (best_match == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
auto ca = SemiInlineGet(best_match->PinCells(op_ctx, {0}));
|
||||
auto index = ca->get_cell_of(0);
|
||||
return PinWrapper<const index::IndexBase*>(std::move(ca),
|
||||
index);
|
||||
});
|
||||
if (res.get() == nullptr) {
|
||||
return {};
|
||||
}
|
||||
return {res};
|
||||
}
|
||||
|
||||
// Mirror of the JsonFlatIndex prefix-matching loop in PinJsonIndex(),
|
||||
// but only reads the nested_path metadata and returns it -- does not
|
||||
// touch the CacheSlot or pin any cell. Callers can use this to decide
|
||||
// whether a JSON query is compatible with the indexed path before
|
||||
// committing to the ScalarIndex exec path (and paying for a pin).
|
||||
std::string
|
||||
GetJsonFlatIndexNestedPath(FieldId field_id,
|
||||
std::string_view query_path) const override {
|
||||
return json_indices.withRLock([&](auto& vec) -> std::string {
|
||||
std::string best_path;
|
||||
int path_len_diff = std::numeric_limits<int>::max();
|
||||
for (const auto& index : vec) {
|
||||
if (index.field_id != field_id) {
|
||||
continue;
|
||||
}
|
||||
if (index.cast_type.data_type() !=
|
||||
JsonCastType::DataType::JSON) {
|
||||
continue;
|
||||
}
|
||||
if (query_path.length() < index.nested_path.length()) {
|
||||
continue;
|
||||
}
|
||||
if (query_path.substr(0, index.nested_path.length()) !=
|
||||
index.nested_path) {
|
||||
continue;
|
||||
}
|
||||
int current_len_diff =
|
||||
query_path.length() - index.nested_path.length();
|
||||
if (current_len_diff < path_len_diff) {
|
||||
path_len_diff = current_len_diff;
|
||||
best_path = index.nested_path;
|
||||
}
|
||||
if (path_len_diff == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return best_path;
|
||||
});
|
||||
}
|
||||
|
||||
virtual PinWrapper<index::NgramInvertedIndex*>
|
||||
GetNgramIndex(milvus::OpContext* op_ctx,
|
||||
FieldId field_id) const override = 0;
|
||||
@@ -178,16 +72,6 @@ class SegmentSealed : public SegmentInternalInterface {
|
||||
type() const override {
|
||||
return SegmentType::Sealed;
|
||||
}
|
||||
|
||||
protected:
|
||||
struct JsonIndex {
|
||||
FieldId field_id;
|
||||
std::string nested_path;
|
||||
JsonCastType cast_type{JsonCastType::UNKNOWN};
|
||||
index::CacheIndexBasePtr index;
|
||||
};
|
||||
|
||||
folly::Synchronized<std::vector<JsonIndex>> json_indices;
|
||||
};
|
||||
|
||||
using SegmentSealedSPtr = std::shared_ptr<SegmentSealed>;
|
||||
|
||||
@@ -788,45 +788,6 @@ LoadDeletedRecord(CSegmentInterface c_segment,
|
||||
}
|
||||
}
|
||||
|
||||
CStatus
|
||||
UpdateSealedSegmentIndex(CSegmentInterface c_segment,
|
||||
CLoadIndexInfo c_load_index_info) {
|
||||
SCOPE_CGO_CALL_METRIC();
|
||||
|
||||
try {
|
||||
auto segment_interface =
|
||||
reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
|
||||
auto segment =
|
||||
dynamic_cast<milvus::segcore::SegmentSealed*>(segment_interface);
|
||||
AssertInfo(segment != nullptr, "segment conversion failed");
|
||||
auto load_index_info =
|
||||
static_cast<milvus::segcore::LoadIndexInfo*>(c_load_index_info);
|
||||
segment->LoadIndex(*load_index_info);
|
||||
return milvus::SuccessCStatus();
|
||||
} catch (std::exception& e) {
|
||||
return milvus::FailureCStatus(&e);
|
||||
}
|
||||
}
|
||||
|
||||
CStatus
|
||||
UpdateFieldRawDataSize(CSegmentInterface c_segment,
|
||||
int64_t field_id,
|
||||
int64_t num_rows,
|
||||
int64_t field_data_size) {
|
||||
SCOPE_CGO_CALL_METRIC();
|
||||
|
||||
try {
|
||||
auto segment_interface =
|
||||
reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
|
||||
AssertInfo(segment_interface != nullptr, "segment conversion failed");
|
||||
segment_interface->set_field_avg_size(
|
||||
milvus::FieldId(field_id), num_rows, field_data_size);
|
||||
return milvus::SuccessCStatus();
|
||||
} catch (std::exception& e) {
|
||||
return milvus::FailureCStatus(&e);
|
||||
}
|
||||
}
|
||||
|
||||
CStatus
|
||||
DropFieldData(CSegmentInterface c_segment, int64_t field_id) {
|
||||
SCOPE_CGO_CALL_METRIC();
|
||||
|
||||
@@ -244,16 +244,6 @@ CStatus
|
||||
LoadDeletedRecord(CSegmentInterface c_segment,
|
||||
CLoadDeletedRecordInfo deleted_record_info);
|
||||
|
||||
CStatus
|
||||
UpdateSealedSegmentIndex(CSegmentInterface c_segment,
|
||||
CLoadIndexInfo c_load_index_info);
|
||||
|
||||
CStatus
|
||||
UpdateFieldRawDataSize(CSegmentInterface c_segment,
|
||||
int64_t field_id,
|
||||
int64_t num_rows,
|
||||
int64_t field_data_size);
|
||||
|
||||
// This function is currently used only in test.
|
||||
// Current implement supports only dropping of non-system fields.
|
||||
CStatus
|
||||
|
||||
@@ -1450,8 +1450,7 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
|
||||
auto segment = CreateSealedWithFieldDataLoaded(schema, dataset);
|
||||
|
||||
// load vec index
|
||||
status = UpdateSealedSegmentIndex(segment.get(), &load_index_info);
|
||||
ASSERT_EQ(status.error_code, Success);
|
||||
segment->LoadIndex(load_index_info);
|
||||
|
||||
CSearchResult c_search_result_on_bigIndex;
|
||||
auto res_after_load_index = CSearch(segment.get(),
|
||||
|
||||
@@ -4702,6 +4702,278 @@ TEST(SealedSegmentCowState, ClearPublishedStateDropsRuntimeSnapshot) {
|
||||
EXPECT_FALSE(GetFieldBit(after->field_data_ready_bitset, payload));
|
||||
}
|
||||
|
||||
TEST(SealedSegmentCowState, JsonIndexStagesAndFollowsSnapshotLifetime) {
|
||||
auto schema = std::make_shared<Schema>();
|
||||
auto pk = schema->AddDebugField("pk", DataType::INT64);
|
||||
auto json = schema->AddDebugField("payload", DataType::JSON);
|
||||
schema->set_primary_field_id(pk);
|
||||
|
||||
auto segment = CreateSealedSegment(schema);
|
||||
auto* sealed = dynamic_cast<ChunkedSegmentSealedImpl*>(segment.get());
|
||||
ASSERT_NE(sealed, nullptr);
|
||||
|
||||
std::array<int64_t, 4> values = {1, 2, 3, 4};
|
||||
auto indexing = GenScalarIndexing<int64_t>(values.size(), values.data());
|
||||
LoadIndexInfo load_info;
|
||||
load_info.field_id = json.get();
|
||||
load_info.field_type = DataType::JSON;
|
||||
load_info.index_params = {
|
||||
{index::INDEX_TYPE, index::INVERTED_INDEX_TYPE},
|
||||
{JSON_PATH, "a"},
|
||||
{JSON_CAST_TYPE, "DOUBLE"},
|
||||
};
|
||||
load_info.cache_index =
|
||||
CreateTestCacheIndex("json-runtime", std::move(indexing));
|
||||
auto loaded_index = load_info.cache_index;
|
||||
|
||||
auto current = sealed->TestGetPublishedStateSnapshot();
|
||||
ASSERT_TRUE(current->runtime->json_indices.empty());
|
||||
auto runtime = sealed->TestCloneMutableRuntimeResourceState();
|
||||
ChunkedSegmentSealedImpl::StateDelta initial_delta;
|
||||
initial_delta.schema = current->schema;
|
||||
initial_delta.load_info = current->load_info;
|
||||
initial_delta.runtime = sealed->TestFreezeRuntimeResourceState(runtime);
|
||||
initial_delta.commit_ts = current->commit_ts;
|
||||
auto staged = sealed->TestBuildNextPublishedState(current, initial_delta);
|
||||
|
||||
ChunkedSegmentSealedImpl::StateDelta final_delta;
|
||||
final_delta.schema = current->schema;
|
||||
final_delta.load_info = current->load_info;
|
||||
final_delta.commit_ts = current->commit_ts;
|
||||
sealed->TestStageLoadIndexThenPublish(
|
||||
load_info,
|
||||
false,
|
||||
schema,
|
||||
runtime,
|
||||
staged.get(),
|
||||
current,
|
||||
final_delta,
|
||||
[&] {
|
||||
EXPECT_EQ(runtime->json_indices.size(), 1);
|
||||
EXPECT_TRUE(current->runtime->json_indices.empty());
|
||||
EXPECT_FALSE(sealed->HasJsonIndex(json));
|
||||
});
|
||||
|
||||
auto published = sealed->TestGetPublishedStateSnapshot();
|
||||
ASSERT_EQ(published->runtime->json_indices.size(), 1);
|
||||
EXPECT_EQ(published->runtime->json_indices.front().index, loaded_index);
|
||||
EXPECT_TRUE(sealed->HasJsonIndex(json));
|
||||
|
||||
sealed->DropJSONIndex(json, "a");
|
||||
auto after_drop = sealed->TestGetPublishedStateSnapshot();
|
||||
EXPECT_TRUE(after_drop->runtime->json_indices.empty());
|
||||
EXPECT_FALSE(sealed->HasJsonIndex(json));
|
||||
|
||||
ASSERT_EQ(published->runtime->json_indices.size(), 1);
|
||||
EXPECT_EQ(published->runtime->json_indices.front().index, loaded_index);
|
||||
}
|
||||
|
||||
TEST(SealedSegmentCowState, JsonIndexReplaceScalarWithNgramErasesScalarPath) {
|
||||
auto schema = std::make_shared<Schema>();
|
||||
auto pk = schema->AddDebugField("pk", DataType::INT64);
|
||||
auto json = schema->AddDebugField("payload", DataType::JSON);
|
||||
schema->set_primary_field_id(pk);
|
||||
|
||||
auto segment = CreateSealedSegment(schema);
|
||||
auto* sealed = dynamic_cast<ChunkedSegmentSealedImpl*>(segment.get());
|
||||
ASSERT_NE(sealed, nullptr);
|
||||
|
||||
std::array<int64_t, 4> values = {1, 2, 3, 4};
|
||||
auto make_load_info = [&](std::string key,
|
||||
std::string path,
|
||||
bool is_ngram) {
|
||||
LoadIndexInfo info;
|
||||
info.field_id = json.get();
|
||||
info.field_type = DataType::JSON;
|
||||
info.index_params = {
|
||||
{index::INDEX_TYPE,
|
||||
is_ngram ? index::NGRAM_INDEX_TYPE : index::INVERTED_INDEX_TYPE},
|
||||
{JSON_PATH, std::move(path)},
|
||||
};
|
||||
if (!is_ngram) {
|
||||
info.index_params[JSON_CAST_TYPE] = "DOUBLE";
|
||||
}
|
||||
auto indexing =
|
||||
GenScalarIndexing<int64_t>(values.size(), values.data());
|
||||
info.cache_index =
|
||||
CreateTestCacheIndex(std::move(key), std::move(indexing));
|
||||
return info;
|
||||
};
|
||||
|
||||
auto sibling = make_load_info("json-scalar-b", "b", false);
|
||||
auto sibling_index = sibling.cache_index;
|
||||
sealed->LoadIndex(sibling);
|
||||
|
||||
auto original = make_load_info("json-scalar-a", "a", false);
|
||||
auto original_index = original.cache_index;
|
||||
sealed->LoadIndex(original);
|
||||
|
||||
auto current = sealed->TestGetPublishedStateSnapshot();
|
||||
ASSERT_EQ(current->runtime->json_indices.size(), 2);
|
||||
EXPECT_TRUE(current->runtime->ngram_indexings.empty());
|
||||
EXPECT_FALSE(GetFieldBit(current->index_ready_bitset, json));
|
||||
|
||||
auto runtime = sealed->TestCloneMutableRuntimeResourceState();
|
||||
ChunkedSegmentSealedImpl::StateDelta initial_delta;
|
||||
initial_delta.schema = current->schema;
|
||||
initial_delta.load_info = current->load_info;
|
||||
initial_delta.runtime = sealed->TestFreezeRuntimeResourceState(runtime);
|
||||
initial_delta.commit_ts = current->commit_ts;
|
||||
auto staged = sealed->TestBuildNextPublishedState(current, initial_delta);
|
||||
|
||||
auto replacement = make_load_info("json-ngram-a", "a", true);
|
||||
auto replacement_index = replacement.cache_index;
|
||||
ChunkedSegmentSealedImpl::StateDelta final_delta;
|
||||
final_delta.schema = current->schema;
|
||||
final_delta.load_info = current->load_info;
|
||||
final_delta.commit_ts = current->commit_ts;
|
||||
sealed->TestStageLoadIndexThenPublish(
|
||||
replacement,
|
||||
true,
|
||||
schema,
|
||||
runtime,
|
||||
staged.get(),
|
||||
current,
|
||||
final_delta,
|
||||
[&] {
|
||||
ASSERT_EQ(runtime->json_indices.size(), 1);
|
||||
EXPECT_EQ(runtime->json_indices.front().nested_path, "b");
|
||||
EXPECT_EQ(runtime->json_indices.front().index, sibling_index);
|
||||
ASSERT_EQ(runtime->ngram_indexings.count(json), 1);
|
||||
ASSERT_EQ(runtime->ngram_indexings.at(json).count("a"), 1);
|
||||
EXPECT_EQ(runtime->ngram_indexings.at(json).at("a"),
|
||||
replacement_index);
|
||||
EXPECT_TRUE(GetFieldBit(staged->index_ready_bitset, json));
|
||||
|
||||
ASSERT_EQ(current->runtime->json_indices.size(), 2);
|
||||
EXPECT_TRUE(current->runtime->ngram_indexings.empty());
|
||||
});
|
||||
|
||||
auto published = sealed->TestGetPublishedStateSnapshot();
|
||||
ASSERT_EQ(published->runtime->json_indices.size(), 1);
|
||||
EXPECT_EQ(published->runtime->json_indices.front().nested_path, "b");
|
||||
EXPECT_EQ(published->runtime->json_indices.front().index, sibling_index);
|
||||
ASSERT_EQ(published->runtime->ngram_indexings.count(json), 1);
|
||||
EXPECT_EQ(published->runtime->ngram_indexings.at(json).at("a"),
|
||||
replacement_index);
|
||||
EXPECT_TRUE(GetFieldBit(published->index_ready_bitset, json));
|
||||
|
||||
ASSERT_EQ(current->runtime->json_indices.size(), 2);
|
||||
auto old_path = std::find_if(
|
||||
current->runtime->json_indices.begin(),
|
||||
current->runtime->json_indices.end(),
|
||||
[](const auto& index) { return index.nested_path == "a"; });
|
||||
ASSERT_NE(old_path, current->runtime->json_indices.end());
|
||||
EXPECT_EQ(old_path->index, original_index);
|
||||
}
|
||||
|
||||
TEST(SealedSegmentCowState, JsonIndexReplaceNgramWithScalarErasesNgramPath) {
|
||||
auto schema = std::make_shared<Schema>();
|
||||
auto pk = schema->AddDebugField("pk", DataType::INT64);
|
||||
auto json = schema->AddDebugField("payload", DataType::JSON);
|
||||
schema->set_primary_field_id(pk);
|
||||
|
||||
auto segment = CreateSealedSegment(schema);
|
||||
auto* sealed = dynamic_cast<ChunkedSegmentSealedImpl*>(segment.get());
|
||||
ASSERT_NE(sealed, nullptr);
|
||||
|
||||
std::array<int64_t, 4> values = {1, 2, 3, 4};
|
||||
auto make_load_info = [&](std::string key,
|
||||
std::string path,
|
||||
bool is_ngram) {
|
||||
LoadIndexInfo info;
|
||||
info.field_id = json.get();
|
||||
info.field_type = DataType::JSON;
|
||||
info.index_params = {
|
||||
{index::INDEX_TYPE,
|
||||
is_ngram ? index::NGRAM_INDEX_TYPE : index::INVERTED_INDEX_TYPE},
|
||||
{JSON_PATH, std::move(path)},
|
||||
};
|
||||
if (!is_ngram) {
|
||||
info.index_params[JSON_CAST_TYPE] = "DOUBLE";
|
||||
}
|
||||
auto indexing =
|
||||
GenScalarIndexing<int64_t>(values.size(), values.data());
|
||||
info.cache_index =
|
||||
CreateTestCacheIndex(std::move(key), std::move(indexing));
|
||||
return info;
|
||||
};
|
||||
|
||||
auto sibling = make_load_info("json-scalar-b", "b", false);
|
||||
auto sibling_index = sibling.cache_index;
|
||||
sealed->LoadIndex(sibling);
|
||||
|
||||
auto original = make_load_info("json-ngram-a", "a", true);
|
||||
auto original_index = original.cache_index;
|
||||
sealed->LoadIndex(original);
|
||||
|
||||
auto current = sealed->TestGetPublishedStateSnapshot();
|
||||
ASSERT_EQ(current->runtime->json_indices.size(), 1);
|
||||
ASSERT_EQ(current->runtime->ngram_indexings.count(json), 1);
|
||||
EXPECT_EQ(current->runtime->ngram_indexings.at(json).at("a"),
|
||||
original_index);
|
||||
EXPECT_TRUE(GetFieldBit(current->index_ready_bitset, json));
|
||||
|
||||
auto runtime = sealed->TestCloneMutableRuntimeResourceState();
|
||||
ChunkedSegmentSealedImpl::StateDelta initial_delta;
|
||||
initial_delta.schema = current->schema;
|
||||
initial_delta.load_info = current->load_info;
|
||||
initial_delta.runtime = sealed->TestFreezeRuntimeResourceState(runtime);
|
||||
initial_delta.commit_ts = current->commit_ts;
|
||||
auto staged = sealed->TestBuildNextPublishedState(current, initial_delta);
|
||||
|
||||
auto replacement = make_load_info("json-scalar-a", "a", false);
|
||||
auto replacement_index = replacement.cache_index;
|
||||
ChunkedSegmentSealedImpl::StateDelta final_delta;
|
||||
final_delta.schema = current->schema;
|
||||
final_delta.load_info = current->load_info;
|
||||
final_delta.commit_ts = current->commit_ts;
|
||||
sealed->TestStageLoadIndexThenPublish(
|
||||
replacement,
|
||||
true,
|
||||
schema,
|
||||
runtime,
|
||||
staged.get(),
|
||||
current,
|
||||
final_delta,
|
||||
[&] {
|
||||
EXPECT_TRUE(runtime->ngram_indexings.empty());
|
||||
ASSERT_EQ(runtime->json_indices.size(), 2);
|
||||
auto replacement_path = std::find_if(
|
||||
runtime->json_indices.begin(),
|
||||
runtime->json_indices.end(),
|
||||
[](const auto& index) { return index.nested_path == "a"; });
|
||||
ASSERT_NE(replacement_path, runtime->json_indices.end());
|
||||
EXPECT_EQ(replacement_path->index, replacement_index);
|
||||
EXPECT_FALSE(GetFieldBit(staged->index_ready_bitset, json));
|
||||
|
||||
ASSERT_EQ(current->runtime->ngram_indexings.count(json), 1);
|
||||
EXPECT_EQ(current->runtime->ngram_indexings.at(json).at("a"),
|
||||
original_index);
|
||||
});
|
||||
|
||||
auto published = sealed->TestGetPublishedStateSnapshot();
|
||||
EXPECT_TRUE(published->runtime->ngram_indexings.empty());
|
||||
ASSERT_EQ(published->runtime->json_indices.size(), 2);
|
||||
auto sibling_path = std::find_if(
|
||||
published->runtime->json_indices.begin(),
|
||||
published->runtime->json_indices.end(),
|
||||
[](const auto& index) { return index.nested_path == "b"; });
|
||||
ASSERT_NE(sibling_path, published->runtime->json_indices.end());
|
||||
EXPECT_EQ(sibling_path->index, sibling_index);
|
||||
auto replacement_path = std::find_if(
|
||||
published->runtime->json_indices.begin(),
|
||||
published->runtime->json_indices.end(),
|
||||
[](const auto& index) { return index.nested_path == "a"; });
|
||||
ASSERT_NE(replacement_path, published->runtime->json_indices.end());
|
||||
EXPECT_EQ(replacement_path->index, replacement_index);
|
||||
EXPECT_FALSE(GetFieldBit(published->index_ready_bitset, json));
|
||||
|
||||
ASSERT_EQ(current->runtime->ngram_indexings.count(json), 1);
|
||||
EXPECT_EQ(current->runtime->ngram_indexings.at(json).at("a"),
|
||||
original_index);
|
||||
}
|
||||
|
||||
TEST(SealedSegmentCowState, JsonStatsLivesInRuntimeSnapshot) {
|
||||
auto schema = std::make_shared<Schema>();
|
||||
auto pk = schema->AddDebugField("pk", DataType::INT64);
|
||||
|
||||
@@ -135,10 +135,6 @@ func packLoadSegmentRequest(
|
||||
indexInfo []*indexpb.IndexInfo,
|
||||
) *querypb.LoadSegmentsRequest {
|
||||
loadScope := querypb.LoadScope_Full
|
||||
if action.Type() == ActionTypeUpdate {
|
||||
loadScope = querypb.LoadScope_Index
|
||||
}
|
||||
|
||||
if action.Type() == ActionTypeStatsUpdate {
|
||||
loadScope = querypb.LoadScope_Stats
|
||||
}
|
||||
|
||||
@@ -573,11 +573,6 @@ func (s *UtilsSuite) TestPackLoadSegmentRequestLoadScope() {
|
||||
action ActionType
|
||||
wantScope querypb.LoadScope
|
||||
}{
|
||||
{
|
||||
name: "index update",
|
||||
action: ActionTypeUpdate,
|
||||
wantScope: querypb.LoadScope_Index,
|
||||
},
|
||||
{
|
||||
name: "legacy stats update",
|
||||
action: ActionTypeStatsUpdate,
|
||||
|
||||
@@ -6,14 +6,10 @@ import (
|
||||
context "context"
|
||||
|
||||
commonpb "github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
|
||||
internalpb "github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
|
||||
|
||||
streamrpc "github.com/milvus-io/milvus/internal/util/streamrpc"
|
||||
internalpb "github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
|
||||
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockWorker is an autogenerated mock type for the Worker type
|
||||
|
||||
@@ -692,10 +692,6 @@ func (sd *shardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSeg
|
||||
return err
|
||||
}
|
||||
|
||||
// load index segment need no stream delete and distribution change
|
||||
if req.GetLoadScope() == querypb.LoadScope_Index {
|
||||
return nil
|
||||
}
|
||||
if req.GetLoadScope() == querypb.LoadScope_Reopen {
|
||||
return sd.handleReopenPostLoad(ctx, req)
|
||||
}
|
||||
|
||||
@@ -141,39 +141,6 @@ func (node *QueryNode) loadDeltaLogs(ctx context.Context, req *querypb.LoadSegme
|
||||
return merr.Success()
|
||||
}
|
||||
|
||||
func (node *QueryNode) loadIndex(ctx context.Context, req *querypb.LoadSegmentsRequest) *commonpb.Status {
|
||||
log := mlog.With(
|
||||
mlog.FieldCollectionID(req.GetCollectionID()),
|
||||
mlog.Int64s("segmentIDs", lo.Map(req.GetInfos(), func(info *querypb.SegmentLoadInfo, _ int) int64 { return info.GetSegmentID() })),
|
||||
)
|
||||
|
||||
status := merr.Success()
|
||||
log.Info(ctx, "start to load index")
|
||||
|
||||
for _, info := range req.GetInfos() {
|
||||
log := mlog.With(mlog.FieldSegmentID(info.GetSegmentID()))
|
||||
segment := node.manager.Segment.GetSealed(info.GetSegmentID())
|
||||
if segment == nil {
|
||||
log.Warn(ctx, "segment not found for load index operation")
|
||||
continue
|
||||
}
|
||||
localSegment, ok := segment.(*segments.LocalSegment)
|
||||
if !ok {
|
||||
log.Warn(ctx, "segment not local for load index opeartion")
|
||||
continue
|
||||
}
|
||||
|
||||
err := node.loader.LoadIndex(ctx, localSegment, info, req.Version)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "failed to load index", mlog.Err(err))
|
||||
status = merr.Status(err)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return status
|
||||
}
|
||||
|
||||
func (node *QueryNode) reopenSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) *commonpb.Status {
|
||||
log := mlog.With(
|
||||
mlog.FieldCollectionID(req.GetCollectionID()),
|
||||
|
||||
@@ -27,15 +27,11 @@ import "C"
|
||||
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus/pkg/v3/metrics"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/cgopb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
|
||||
)
|
||||
|
||||
// LoadIndexInfo is a wrapper of the underlying C-structure C.CLoadIndexInfo
|
||||
@@ -66,15 +62,6 @@ func deleteLoadIndexInfo(info *LoadIndexInfo) {
|
||||
}).Await()
|
||||
}
|
||||
|
||||
func (li *LoadIndexInfo) cleanLocalData(ctx context.Context) error {
|
||||
var status C.CStatus
|
||||
GetDynamicPool().Submit(func() (any, error) {
|
||||
status = C.CleanLoadedIndex(li.cLoadIndexInfo)
|
||||
return nil, nil
|
||||
}).Await()
|
||||
return HandleCStatus(ctx, &status, "failed to clean cached data on disk")
|
||||
}
|
||||
|
||||
func (li *LoadIndexInfo) appendLoadIndexInfo(ctx context.Context, info *cgopb.LoadIndexInfo) error {
|
||||
marshaled, err := proto.Marshal(info)
|
||||
if err != nil {
|
||||
@@ -98,23 +85,3 @@ func (li *LoadIndexInfo) setShard(shard string) {
|
||||
defer C.free(unsafe.Pointer(cShard))
|
||||
C.SetLoadIndexInfoShard(li.cLoadIndexInfo, cShard)
|
||||
}
|
||||
|
||||
func (li *LoadIndexInfo) loadIndex(ctx context.Context) error {
|
||||
var status C.CStatus
|
||||
_, _ = GetLoadPool().Submit(func() (any, error) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
metrics.QueryNodeCGOCallLatency.WithLabelValues(
|
||||
paramtable.GetStringNodeID(),
|
||||
"AppendIndexV2",
|
||||
"Sync",
|
||||
).Observe(float64(time.Since(start).Milliseconds()))
|
||||
}()
|
||||
traceCtx := ParseCTraceContext(ctx)
|
||||
status = C.AppendIndexV2(traceCtx.ctx, li.cLoadIndexInfo)
|
||||
runtime.KeepAlive(traceCtx)
|
||||
return nil, nil
|
||||
}).Await()
|
||||
|
||||
return HandleCStatus(ctx, &status, "AppendIndexV2 failed")
|
||||
}
|
||||
|
||||
@@ -6,14 +6,10 @@ import (
|
||||
context "context"
|
||||
|
||||
commonpb "github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
pkoracle "github.com/milvus-io/milvus/internal/querynodev2/pkoracle"
|
||||
|
||||
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
|
||||
|
||||
storage "github.com/milvus-io/milvus/internal/storage"
|
||||
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockLoader is an autogenerated mock type for the Loader type
|
||||
@@ -274,55 +270,6 @@ func (_c *MockLoader_LoadDeltaLogs_Call) RunAndReturn(run func(context.Context,
|
||||
return _c
|
||||
}
|
||||
|
||||
// LoadIndex provides a mock function with given fields: ctx, segment, info, version
|
||||
func (_m *MockLoader) LoadIndex(ctx context.Context, segment Segment, info *querypb.SegmentLoadInfo, version int64) error {
|
||||
ret := _m.Called(ctx, segment, info, version)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for LoadIndex")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, Segment, *querypb.SegmentLoadInfo, int64) error); ok {
|
||||
r0 = rf(ctx, segment, info, version)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockLoader_LoadIndex_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadIndex'
|
||||
type MockLoader_LoadIndex_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// LoadIndex is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - segment Segment
|
||||
// - info *querypb.SegmentLoadInfo
|
||||
// - version int64
|
||||
func (_e *MockLoader_Expecter) LoadIndex(ctx interface{}, segment interface{}, info interface{}, version interface{}) *MockLoader_LoadIndex_Call {
|
||||
return &MockLoader_LoadIndex_Call{Call: _e.mock.On("LoadIndex", ctx, segment, info, version)}
|
||||
}
|
||||
|
||||
func (_c *MockLoader_LoadIndex_Call) Run(run func(ctx context.Context, segment Segment, info *querypb.SegmentLoadInfo, version int64)) *MockLoader_LoadIndex_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(Segment), args[2].(*querypb.SegmentLoadInfo), args[3].(int64))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLoader_LoadIndex_Call) Return(_a0 error) *MockLoader_LoadIndex_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockLoader_LoadIndex_Call) RunAndReturn(run func(context.Context, Segment, *querypb.SegmentLoadInfo, int64) error) *MockLoader_LoadIndex_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// ReopenSegments provides a mock function with given fields: ctx, loadInfos
|
||||
func (_m *MockLoader) ReopenSegments(ctx context.Context, loadInfos []*querypb.SegmentLoadInfo) error {
|
||||
ret := _m.Called(ctx, loadInfos)
|
||||
|
||||
@@ -6,10 +6,8 @@ import (
|
||||
context "context"
|
||||
|
||||
commonpb "github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
querypb "github.com/milvus-io/milvus/pkg/v3/proto/querypb"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// MockSegmentManager is an autogenerated mock type for the SegmentManager type
|
||||
@@ -753,6 +751,40 @@ func (_c *MockSegmentManager_Put_Call) RunAndReturn(run func(context.Context, co
|
||||
return _c
|
||||
}
|
||||
|
||||
// ReleaseDetached provides a mock function with given fields: ctx, segment
|
||||
func (_m *MockSegmentManager) ReleaseDetached(ctx context.Context, segment Segment) {
|
||||
_m.Called(ctx, segment)
|
||||
}
|
||||
|
||||
// MockSegmentManager_ReleaseDetached_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseDetached'
|
||||
type MockSegmentManager_ReleaseDetached_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ReleaseDetached is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - segment Segment
|
||||
func (_e *MockSegmentManager_Expecter) ReleaseDetached(ctx interface{}, segment interface{}) *MockSegmentManager_ReleaseDetached_Call {
|
||||
return &MockSegmentManager_ReleaseDetached_Call{Call: _e.mock.On("ReleaseDetached", ctx, segment)}
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_ReleaseDetached_Call) Run(run func(ctx context.Context, segment Segment)) *MockSegmentManager_ReleaseDetached_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(Segment))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_ReleaseDetached_Call) Return() *MockSegmentManager_ReleaseDetached_Call {
|
||||
_c.Call.Return()
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_ReleaseDetached_Call) RunAndReturn(run func(context.Context, Segment)) *MockSegmentManager_ReleaseDetached_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Remove provides a mock function with given fields: ctx, segmentID, scope
|
||||
func (_m *MockSegmentManager) Remove(ctx context.Context, segmentID int64, scope querypb.DataScope) (int, int) {
|
||||
ret := _m.Called(ctx, segmentID, scope)
|
||||
@@ -811,98 +843,6 @@ func (_c *MockSegmentManager_Remove_Call) RunAndReturn(run func(context.Context,
|
||||
return _c
|
||||
}
|
||||
|
||||
// Detach provides a mock function with given fields: ctx, segmentID, scope
|
||||
func (_m *MockSegmentManager) Detach(ctx context.Context, segmentID int64, scope querypb.DataScope) (int, int) {
|
||||
ret := _m.Called(ctx, segmentID, scope)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Detach")
|
||||
}
|
||||
|
||||
var r0 int
|
||||
var r1 int
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, querypb.DataScope) (int, int)); ok {
|
||||
return rf(ctx, segmentID, scope)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, querypb.DataScope) int); ok {
|
||||
r0 = rf(ctx, segmentID, scope)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, querypb.DataScope) int); ok {
|
||||
r1 = rf(ctx, segmentID, scope)
|
||||
} else {
|
||||
r1 = ret.Get(1).(int)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockSegmentManager_Detach_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Detach'
|
||||
type MockSegmentManager_Detach_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Detach is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - segmentID int64
|
||||
// - scope querypb.DataScope
|
||||
func (_e *MockSegmentManager_Expecter) Detach(ctx interface{}, segmentID interface{}, scope interface{}) *MockSegmentManager_Detach_Call {
|
||||
return &MockSegmentManager_Detach_Call{Call: _e.mock.On("Detach", ctx, segmentID, scope)}
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_Detach_Call) Run(run func(ctx context.Context, segmentID int64, scope querypb.DataScope)) *MockSegmentManager_Detach_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(querypb.DataScope))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_Detach_Call) Return(_a0 int, _a1 int) *MockSegmentManager_Detach_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_Detach_Call) RunAndReturn(run func(context.Context, int64, querypb.DataScope) (int, int)) *MockSegmentManager_Detach_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// ReleaseDetached provides a mock function with given fields: ctx, segment
|
||||
func (_m *MockSegmentManager) ReleaseDetached(ctx context.Context, segment Segment) {
|
||||
_m.Called(ctx, segment)
|
||||
}
|
||||
|
||||
// MockSegmentManager_ReleaseDetached_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseDetached'
|
||||
type MockSegmentManager_ReleaseDetached_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ReleaseDetached is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - segment Segment
|
||||
func (_e *MockSegmentManager_Expecter) ReleaseDetached(ctx interface{}, segment interface{}) *MockSegmentManager_ReleaseDetached_Call {
|
||||
return &MockSegmentManager_ReleaseDetached_Call{Call: _e.mock.On("ReleaseDetached", ctx, segment)}
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_ReleaseDetached_Call) Run(run func(ctx context.Context, segment Segment)) *MockSegmentManager_ReleaseDetached_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(Segment))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_ReleaseDetached_Call) Return() *MockSegmentManager_ReleaseDetached_Call {
|
||||
_c.Call.Return()
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSegmentManager_ReleaseDetached_Call) RunAndReturn(run func(context.Context, Segment)) *MockSegmentManager_ReleaseDetached_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// RemoveBy provides a mock function with given fields: ctx, filters
|
||||
func (_m *MockSegmentManager) RemoveBy(ctx context.Context, filters ...SegmentFilter) (int, int) {
|
||||
_va := make([]interface{}, len(filters))
|
||||
|
||||
@@ -1242,153 +1242,6 @@ func GetCLoadInfoWithFunc(ctx context.Context,
|
||||
return f(loadIndexInfo)
|
||||
}
|
||||
|
||||
func (s *LocalSegment) LoadIndex(ctx context.Context, indexInfo *querypb.FieldIndexInfo, fieldType schemapb.DataType) error {
|
||||
log := mlog.With(
|
||||
mlog.FieldCollectionID(s.Collection()),
|
||||
mlog.FieldPartitionID(s.Partition()),
|
||||
mlog.FieldSegmentID(s.ID()),
|
||||
mlog.FieldFieldID(indexInfo.GetFieldID()),
|
||||
mlog.FieldIndexID(indexInfo.GetIndexID()),
|
||||
)
|
||||
|
||||
old := s.GetIndexByID(indexInfo.GetIndexID())
|
||||
// the index loaded
|
||||
if old != nil && old.IsLoaded {
|
||||
log.Warn(ctx, "index already loaded")
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, sp := otel.Tracer(typeutil.QueryNodeRole).Start(ctx, fmt.Sprintf("LoadIndex-%d-%d", s.ID(), indexInfo.GetFieldID()))
|
||||
defer sp.End()
|
||||
|
||||
tr := timerecord.NewTimeRecorder("loadIndex")
|
||||
|
||||
schemaHelper, err := typeutil.CreateSchemaHelper(s.GetCollection().Schema())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fieldSchema, err := schemaHelper.GetFieldFromID(indexInfo.GetFieldID())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// // if segment is pk sorted, user created indexes bring no performance gain but extra memory usage
|
||||
if s.IsSorted() && fieldSchema.GetIsPrimaryKey() {
|
||||
log.Info(ctx, "skip loading index for pk field in sorted segment")
|
||||
// set field index, preventing repeated loading index task
|
||||
s.fieldIndexes.Insert(indexInfo.GetFieldID(), &IndexedFieldInfo{
|
||||
FieldBinlog: &datapb.FieldBinlog{
|
||||
FieldID: indexInfo.GetFieldID(),
|
||||
},
|
||||
IndexInfo: indexInfo,
|
||||
IsLoaded: true,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.innerLoadIndex(ctx, fieldSchema, indexInfo, tr, fieldType)
|
||||
}
|
||||
|
||||
func (s *LocalSegment) innerLoadIndex(ctx context.Context,
|
||||
fieldSchema *schemapb.FieldSchema,
|
||||
indexInfo *querypb.FieldIndexInfo,
|
||||
tr *timerecord.TimeRecorder,
|
||||
fieldType schemapb.DataType,
|
||||
) error {
|
||||
err := GetCLoadInfoWithFunc(ctx, fieldSchema,
|
||||
s.LoadInfo(), indexInfo, func(loadIndexInfo *LoadIndexInfo) error {
|
||||
newLoadIndexInfoSpan := tr.RecordSpan()
|
||||
|
||||
if err := loadIndexInfo.loadIndex(ctx); err != nil {
|
||||
if loadIndexInfo.cleanLocalData(ctx) != nil {
|
||||
mlog.Warn(ctx, "failed to clean cached data on disk after append index failed",
|
||||
mlog.FieldBuildID(indexInfo.BuildID),
|
||||
mlog.Int64("index version", indexInfo.IndexVersion))
|
||||
}
|
||||
return err
|
||||
}
|
||||
if s.Type() != SegmentTypeSealed {
|
||||
return merr.WrapErrServiceInternalMsg("updateSegmentIndex failed, illegal segment type %s, segmentID = %d", s.segmentType, s.ID())
|
||||
}
|
||||
appendLoadIndexInfoSpan := tr.RecordSpan()
|
||||
|
||||
// 3.
|
||||
err := s.UpdateIndexInfo(ctx, indexInfo, loadIndexInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
updateIndexInfoSpan := tr.RecordSpan()
|
||||
|
||||
mlog.Info(ctx, "Finish loading index",
|
||||
mlog.Duration("newLoadIndexInfoSpan", newLoadIndexInfoSpan),
|
||||
mlog.Duration("appendLoadIndexInfoSpan", appendLoadIndexInfoSpan),
|
||||
mlog.Duration("updateIndexInfoSpan", updateIndexInfoSpan),
|
||||
)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
mlog.Warn(ctx, "load index failed", mlog.Err(err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *LocalSegment) UpdateIndexInfo(ctx context.Context, indexInfo *querypb.FieldIndexInfo, info *LoadIndexInfo) error {
|
||||
log := mlog.With(
|
||||
mlog.FieldCollectionID(s.Collection()),
|
||||
mlog.FieldPartitionID(s.Partition()),
|
||||
mlog.FieldSegmentID(s.ID()),
|
||||
mlog.FieldFieldID(indexInfo.FieldID),
|
||||
)
|
||||
if !s.ptrLock.PinIf(state.IsNotReleased) {
|
||||
return merr.WrapErrSegmentNotLoaded(s.ID(), "segment released")
|
||||
}
|
||||
defer s.ptrLock.Unpin()
|
||||
|
||||
var status C.CStatus
|
||||
GetDynamicPool().Submit(func() (any, error) {
|
||||
status = C.UpdateSealedSegmentIndex(s.ptr, info.cLoadIndexInfo)
|
||||
return nil, nil
|
||||
}).Await()
|
||||
|
||||
if err := HandleCStatus(ctx, &status, "UpdateSealedSegmentIndex failed",
|
||||
mlog.FieldCollectionID(s.Collection()),
|
||||
mlog.FieldPartitionID(s.Partition()),
|
||||
mlog.FieldSegmentID(s.ID()),
|
||||
mlog.FieldFieldID(indexInfo.FieldID)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.fieldIndexes.Insert(indexInfo.GetIndexID(), &IndexedFieldInfo{
|
||||
FieldBinlog: &datapb.FieldBinlog{
|
||||
FieldID: indexInfo.GetFieldID(),
|
||||
},
|
||||
IndexInfo: indexInfo,
|
||||
IsLoaded: true,
|
||||
})
|
||||
log.Info(ctx, "updateSegmentIndex done")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LocalSegment) UpdateFieldRawDataSize(ctx context.Context, numRows int64, fieldBinlog *datapb.FieldBinlog) error {
|
||||
var status C.CStatus
|
||||
fieldID := fieldBinlog.FieldID
|
||||
fieldDataSize := int64(0)
|
||||
for _, binlog := range fieldBinlog.GetBinlogs() {
|
||||
fieldDataSize += binlog.GetMemorySize()
|
||||
}
|
||||
GetDynamicPool().Submit(func() (any, error) {
|
||||
status = C.UpdateFieldRawDataSize(s.ptr, C.int64_t(fieldID), C.int64_t(numRows), C.int64_t(fieldDataSize))
|
||||
return nil, nil
|
||||
}).Await()
|
||||
|
||||
if err := HandleCStatus(ctx, &status, "updateFieldRawDataSize failed"); err != nil {
|
||||
return err
|
||||
}
|
||||
mlog.Info(ctx, "updateFieldRawDataSize done", mlog.FieldSegmentID(s.ID()))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *LocalSegment) syncFieldJSONStatsFromLoadInfo(ctx context.Context, loadInfo *querypb.SegmentLoadInfo) {
|
||||
jsonStatsInfo := make(map[int64]*querypb.JsonStatsInfo)
|
||||
if !paramtable.Get().CommonCfg.EnabledJSONKeyStats.GetAsBool() {
|
||||
|
||||
@@ -90,12 +90,6 @@ type Loader interface {
|
||||
// GetChunkManager returns the chunk manager for remote storage access.
|
||||
GetChunkManager() storage.ChunkManager
|
||||
|
||||
// LoadIndex append index for segment and remove vector binlogs.
|
||||
LoadIndex(ctx context.Context,
|
||||
segment Segment,
|
||||
info *querypb.SegmentLoadInfo,
|
||||
version int64) error
|
||||
|
||||
// ReopenSegments update segment data according to new load info.
|
||||
ReopenSegments(ctx context.Context,
|
||||
loadInfos []*querypb.SegmentLoadInfo,
|
||||
@@ -1198,45 +1192,6 @@ func loadSealedSegmentFields(ctx context.Context, collection *Collection, segmen
|
||||
return nil
|
||||
}
|
||||
|
||||
func (loader *segmentLoader) loadFieldsIndex(ctx context.Context,
|
||||
schemaHelper *typeutil.SchemaHelper,
|
||||
segment *LocalSegment,
|
||||
numRows int64,
|
||||
indexedFieldInfos map[int64]*IndexedFieldInfo,
|
||||
) error {
|
||||
for _, fieldInfo := range indexedFieldInfos {
|
||||
fieldID := fieldInfo.IndexInfo.FieldID
|
||||
indexInfo := fieldInfo.IndexInfo
|
||||
tr := timerecord.NewTimeRecorder("loadFieldIndex")
|
||||
err := loader.loadFieldIndex(ctx, segment, indexInfo)
|
||||
loadFieldIndexSpan := tr.RecordSpan()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mlog.Info(context.TODO(), "load field binlogs done for sealed segment with index",
|
||||
mlog.Int64("fieldID", fieldID),
|
||||
mlog.Any("binlog", fieldInfo.FieldBinlog.Binlogs),
|
||||
mlog.Int32("current_index_version", fieldInfo.IndexInfo.GetCurrentIndexVersion()),
|
||||
mlog.Duration("load_duration", loadFieldIndexSpan),
|
||||
)
|
||||
|
||||
// set average row data size of variable field
|
||||
field, err := schemaHelper.GetFieldFromID(fieldID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if typeutil.IsVariableDataType(field.GetDataType()) {
|
||||
err = segment.UpdateFieldRawDataSize(ctx, numRows, fieldInfo.FieldBinlog)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (loader *segmentLoader) loadBm25Stats(ctx context.Context, segmentID int64, stats map[int64]*storage.BM25Stats, binlogPaths map[int64][]string) error {
|
||||
if len(binlogPaths) == 0 {
|
||||
mlog.Info(context.TODO(), "there are no bm25 stats logs saved with segment")
|
||||
@@ -1279,29 +1234,6 @@ func (loader *segmentLoader) loadBm25Stats(ctx context.Context, segmentID int64,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (loader *segmentLoader) loadFieldIndex(ctx context.Context, segment *LocalSegment, indexInfo *querypb.FieldIndexInfo) error {
|
||||
filteredPaths := make([]string, 0, len(indexInfo.IndexFilePaths))
|
||||
|
||||
for _, indexPath := range indexInfo.IndexFilePaths {
|
||||
if path.Base(indexPath) != storage.IndexParamsKey {
|
||||
filteredPaths = append(filteredPaths, indexPath)
|
||||
}
|
||||
}
|
||||
|
||||
indexInfo.IndexFilePaths = filteredPaths
|
||||
fieldType, err := loader.getFieldType(segment.Collection(), indexInfo.FieldID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
collection := loader.manager.Collection.Get(segment.Collection())
|
||||
if collection == nil {
|
||||
return merr.WrapErrCollectionNotLoaded(segment.Collection(), "failed to load field index")
|
||||
}
|
||||
|
||||
return segment.LoadIndex(ctx, indexInfo, fieldType)
|
||||
}
|
||||
|
||||
func (loader *segmentLoader) loadBloomFilter(
|
||||
ctx context.Context,
|
||||
segmentID int64,
|
||||
@@ -2492,92 +2424,6 @@ func SupportInterimIndexDataType(dataType schemapb.DataType) bool {
|
||||
dataType == schemapb.DataType_BFloat16Vector
|
||||
}
|
||||
|
||||
func (loader *segmentLoader) getFieldType(collectionID, fieldID int64) (schemapb.DataType, error) {
|
||||
collection := loader.manager.Collection.Get(collectionID)
|
||||
if collection == nil {
|
||||
return 0, merr.WrapErrCollectionNotFound(collectionID)
|
||||
}
|
||||
|
||||
for _, field := range collection.Schema().GetFields() {
|
||||
if field.GetFieldID() == fieldID {
|
||||
return field.GetDataType(), nil
|
||||
}
|
||||
}
|
||||
|
||||
for _, structField := range collection.Schema().GetStructArrayFields() {
|
||||
if structField.GetFieldID() == fieldID {
|
||||
return schemapb.DataType_ArrayOfStruct, nil
|
||||
}
|
||||
for _, subField := range structField.GetFields() {
|
||||
if subField.GetFieldID() == fieldID {
|
||||
return subField.GetDataType(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0, merr.WrapErrFieldNotFound(fieldID)
|
||||
}
|
||||
|
||||
func (loader *segmentLoader) LoadIndex(ctx context.Context,
|
||||
seg Segment,
|
||||
loadInfo *querypb.SegmentLoadInfo,
|
||||
version int64,
|
||||
) error {
|
||||
segment, ok := seg.(*LocalSegment)
|
||||
if !ok {
|
||||
return merr.WrapErrParameterInvalid("LocalSegment", fmt.Sprintf("%T", seg))
|
||||
}
|
||||
// Filter out LOADING segments only
|
||||
// use None to avoid loaded check
|
||||
infos := loader.prepare(ctx, commonpb.SegmentState_SegmentStateNone, loadInfo)
|
||||
defer loader.unregister(infos...)
|
||||
|
||||
indexInfo := lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
|
||||
info = typeutil.Clone(info)
|
||||
// remain binlog paths whose field id is in index infos to estimate resource usage correctly
|
||||
indexFields := typeutil.NewSet(lo.Map(info.GetIndexInfos(), func(indexInfo *querypb.FieldIndexInfo, _ int) int64 { return indexInfo.GetFieldID() })...)
|
||||
var binlogPaths []*datapb.FieldBinlog
|
||||
for _, binlog := range info.GetBinlogPaths() {
|
||||
if indexFields.Contain(binlog.GetFieldID()) {
|
||||
binlogPaths = append(binlogPaths, binlog)
|
||||
}
|
||||
}
|
||||
info.BinlogPaths = binlogPaths
|
||||
info.Deltalogs = nil
|
||||
info.Statslogs = nil
|
||||
return info
|
||||
})
|
||||
requestResourceResult, err := loader.requestResource(ctx, indexInfo...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer loader.freeRequestResource(requestResourceResult)
|
||||
|
||||
mlog.Info(context.TODO(), "segment loader start to load index", mlog.Int("segmentNumAfterFilter", len(infos)))
|
||||
metrics.QueryNodeLoadSegmentConcurrency.WithLabelValues(paramtable.GetStringNodeID(), "LoadIndex").Inc()
|
||||
defer metrics.QueryNodeLoadSegmentConcurrency.WithLabelValues(paramtable.GetStringNodeID(), "LoadIndex").Dec()
|
||||
|
||||
tr := timerecord.NewTimeRecorder("segmentLoader.LoadIndex")
|
||||
defer metrics.QueryNodeLoadIndexLatency.WithLabelValues(paramtable.GetStringNodeID()).Observe(float64(tr.ElapseSpan().Milliseconds()))
|
||||
for _, loadInfo := range infos {
|
||||
for _, info := range loadInfo.GetIndexInfos() {
|
||||
if len(info.GetIndexFilePaths()) == 0 {
|
||||
mlog.Warn(context.TODO(), "failed to add index for segment, index file list is empty, the segment may be too small")
|
||||
return merr.WrapErrIndexNotFound("index file list empty")
|
||||
}
|
||||
|
||||
err := loader.loadFieldIndex(ctx, segment, info)
|
||||
if err != nil {
|
||||
mlog.Warn(context.TODO(), "failed to load index for segment", mlog.Err(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
loader.notifyLoadFinish(loadInfo)
|
||||
}
|
||||
|
||||
return loader.waitSegmentLoadDone(ctx, commonpb.SegmentState_SegmentStateNone, []int64{loadInfo.GetSegmentID()}, version)
|
||||
}
|
||||
|
||||
func (loader *segmentLoader) ReopenSegments(ctx context.Context,
|
||||
loadInfos []*querypb.SegmentLoadInfo,
|
||||
) error {
|
||||
|
||||
@@ -35,7 +35,6 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/mocks/util/mock_segcore"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/internal/storagev2/packed"
|
||||
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
|
||||
"github.com/milvus-io/milvus/internal/util/initcore"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/mlog"
|
||||
@@ -1397,69 +1396,6 @@ func (suite *SegmentLoaderSuite) TestLoadDeltaLogsChildManifestReadError() {
|
||||
suite.Contains(err.Error(), "manifest read failed")
|
||||
}
|
||||
|
||||
func (suite *SegmentLoaderSuite) TestLoadIndex() {
|
||||
ctx := context.Background()
|
||||
loadInfo := &querypb.SegmentLoadInfo{
|
||||
SegmentID: 1,
|
||||
PartitionID: suite.partitionID,
|
||||
CollectionID: suite.collectionID,
|
||||
IndexInfos: []*querypb.FieldIndexInfo{
|
||||
{
|
||||
IndexFilePaths: []string{},
|
||||
},
|
||||
},
|
||||
InsertChannel: fmt.Sprintf("by-dev-rootcoord-dml_0_%dv0", suite.collectionID),
|
||||
}
|
||||
segment := &LocalSegment{
|
||||
baseSegment: baseSegment{loadInfo: atomic.NewPointer[querypb.SegmentLoadInfo](loadInfo)},
|
||||
bm25StatsHolder: newBM25StatsHolder(),
|
||||
}
|
||||
|
||||
err := suite.loader.LoadIndex(ctx, segment, loadInfo, 0)
|
||||
suite.ErrorIs(err, merr.ErrIndexNotFound)
|
||||
}
|
||||
|
||||
func (suite *SegmentLoaderSuite) TestLoadIndexWithLimitedResource() {
|
||||
ctx := context.Background()
|
||||
loadInfo := &querypb.SegmentLoadInfo{
|
||||
SegmentID: 1,
|
||||
PartitionID: suite.partitionID,
|
||||
CollectionID: suite.collectionID,
|
||||
IndexInfos: []*querypb.FieldIndexInfo{
|
||||
{
|
||||
FieldID: 1,
|
||||
IndexFilePaths: []string{},
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.IndexINVERTED,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
BinlogPaths: []*datapb.FieldBinlog{
|
||||
{
|
||||
FieldID: 1,
|
||||
Binlogs: []*datapb.Binlog{
|
||||
{
|
||||
LogSize: 1000000000,
|
||||
MemorySize: 1000000000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
segment := &LocalSegment{
|
||||
baseSegment: baseSegment{loadInfo: atomic.NewPointer[querypb.SegmentLoadInfo](loadInfo)},
|
||||
bm25StatsHolder: newBM25StatsHolder(),
|
||||
}
|
||||
paramtable.Get().Save(paramtable.Get().QueryNodeCfg.DiskCapacityLimit.Key, "100000")
|
||||
defer paramtable.Get().Reset(paramtable.Get().QueryNodeCfg.DiskCapacityLimit.Key)
|
||||
err := suite.loader.LoadIndex(ctx, segment, loadInfo, 0)
|
||||
suite.Error(err)
|
||||
}
|
||||
|
||||
func (suite *SegmentLoaderSuite) TestLoadWithMmap() {
|
||||
key := paramtable.Get().QueryNodeCfg.MmapDirPath.Key
|
||||
paramtable.Get().Save(key, "/tmp/mmap-test")
|
||||
|
||||
@@ -66,6 +66,12 @@ import (
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
// legacyLoadScopeIndex is the retired wire value emitted by QueryCoord
|
||||
// versions that predate LoadScope_Reopen. Keep the value out of the proto enum
|
||||
// so it cannot be reused, while recognizing it here to return a precise
|
||||
// mixed-version protocol error.
|
||||
const legacyLoadScopeIndex = querypb.LoadScope(2)
|
||||
|
||||
type segmentDetacher interface {
|
||||
DetachStreaming(ctx context.Context, segmentID typeutil.UniqueID) int
|
||||
}
|
||||
@@ -536,12 +542,21 @@ func (node *QueryNode) LoadSegments(ctx context.Context, req *querypb.LoadSegmen
|
||||
switch req.GetLoadScope() {
|
||||
case querypb.LoadScope_Delta:
|
||||
return node.loadDeltaLogs(ctx, req), nil
|
||||
case querypb.LoadScope_Index:
|
||||
return node.loadIndex(ctx, req), nil
|
||||
case querypb.LoadScope_Stats:
|
||||
return node.reopenSegments(ctx, req), nil
|
||||
case querypb.LoadScope_Reopen:
|
||||
return node.reopenSegments(ctx, req), nil
|
||||
case querypb.LoadScope_Full:
|
||||
// Continue with the full segment load below.
|
||||
case legacyLoadScopeIndex:
|
||||
err := merr.WrapErrServiceInternalMsg(
|
||||
"legacy segment index load scope %d is no longer supported; upgrade QueryCoord to use LoadScope_Reopen before routing requests to this QueryNode",
|
||||
req.GetLoadScope())
|
||||
return merr.Status(err), nil
|
||||
default:
|
||||
err := merr.WrapErrServiceInternalMsg(
|
||||
"unsupported segment load scope %d", req.GetLoadScope())
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
|
||||
// Actual load segment
|
||||
|
||||
@@ -840,24 +840,10 @@ func (suite *ServiceSuite) TestLoadDeltaVarchar() {
|
||||
suite.Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
|
||||
}
|
||||
|
||||
func (suite *ServiceSuite) TestLoadIndex_Success() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
schema := mock_segcore.GenTestCollectionSchema(suite.collectionName, schemapb.DataType_Int64, false)
|
||||
|
||||
indexInfos := mock_segcore.GenTestIndexInfoList(suite.collectionID, schema)
|
||||
infos := suite.genSegmentLoadInfos(schema, indexInfos)
|
||||
infos = lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
|
||||
info.SegmentID = info.SegmentID + 1000
|
||||
return info
|
||||
})
|
||||
rawInfo := lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
|
||||
info = typeutil.Clone(info)
|
||||
info.IndexInfos = nil
|
||||
return info
|
||||
})
|
||||
|
||||
func (suite *ServiceSuite) TestLoadSegmentsRejectsUnknownScope() {
|
||||
ctx := context.Background()
|
||||
schema := mock_segcore.GenTestCollectionSchema(
|
||||
suite.collectionName, schemapb.DataType_Int64, false)
|
||||
req := &querypb.LoadSegmentsRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgID: rand.Int63(),
|
||||
@@ -865,123 +851,46 @@ func (suite *ServiceSuite) TestLoadIndex_Success() {
|
||||
},
|
||||
CollectionID: suite.collectionID,
|
||||
DstNodeID: suite.node.session.ServerID,
|
||||
Infos: rawInfo,
|
||||
Infos: suite.genSegmentLoadInfos(schema, nil),
|
||||
Schema: schema,
|
||||
NeedTransfer: false,
|
||||
LoadScope: querypb.LoadScope_Full,
|
||||
IndexInfoList: indexInfos,
|
||||
LoadScope: querypb.LoadScope(99),
|
||||
IndexInfoList: []*indexpb.IndexInfo{{}},
|
||||
}
|
||||
|
||||
// Load segment
|
||||
status, err := suite.node.LoadSegments(ctx, req)
|
||||
suite.Require().NoError(err)
|
||||
suite.Require().Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
|
||||
suite.NoError(err)
|
||||
suite.NotEqual(commonpb.ErrorCode_Success, status.GetErrorCode())
|
||||
suite.Contains(status.GetReason(), "unsupported segment load scope")
|
||||
}
|
||||
|
||||
for _, segmentID := range lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) int64 {
|
||||
return info.GetSegmentID()
|
||||
}) {
|
||||
suite.Equal(0, len(suite.node.manager.Segment.Get(segmentID).Indexes()))
|
||||
}
|
||||
|
||||
req = &querypb.LoadSegmentsRequest{
|
||||
func (suite *ServiceSuite) TestLoadSegmentsRejectsLegacyIndexScope() {
|
||||
ctx := context.Background()
|
||||
schema := mock_segcore.GenTestCollectionSchema(
|
||||
suite.collectionName, schemapb.DataType_Int64, false)
|
||||
req := &querypb.LoadSegmentsRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgID: rand.Int63(),
|
||||
TargetID: suite.node.session.ServerID,
|
||||
},
|
||||
CollectionID: suite.collectionID,
|
||||
DstNodeID: suite.node.session.ServerID,
|
||||
Infos: infos,
|
||||
Infos: suite.genSegmentLoadInfos(schema, nil),
|
||||
Schema: schema,
|
||||
NeedTransfer: false,
|
||||
LoadScope: querypb.LoadScope_Index,
|
||||
LoadScope: legacyLoadScopeIndex,
|
||||
IndexInfoList: []*indexpb.IndexInfo{{}},
|
||||
}
|
||||
|
||||
// Load segment
|
||||
status, err = suite.node.LoadSegments(ctx, req)
|
||||
suite.Require().NoError(err)
|
||||
suite.Require().Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
|
||||
loader := suite.node.loader
|
||||
suite.node.loader = segments.NewMockLoader(suite.T())
|
||||
defer func() { suite.node.loader = loader }()
|
||||
|
||||
for _, segmentID := range lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) int64 {
|
||||
return info.GetSegmentID()
|
||||
}) {
|
||||
suite.T().Log(segmentID)
|
||||
suite.T().Log(len(suite.node.manager.Segment.Get(segmentID).Indexes()))
|
||||
suite.Greater(len(suite.node.manager.Segment.Get(segmentID).Indexes()), 0)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *ServiceSuite) TestLoadIndex_Failed() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
schema := mock_segcore.GenTestCollectionSchema(suite.collectionName, schemapb.DataType_Int64, false)
|
||||
|
||||
suite.Run("load_non_exist_segment", func() {
|
||||
indexInfos := mock_segcore.GenTestIndexInfoList(suite.collectionID, schema)
|
||||
infos := suite.genSegmentLoadInfos(schema, indexInfos)
|
||||
infos = lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
|
||||
info.SegmentID = info.SegmentID + 1000
|
||||
return info
|
||||
})
|
||||
rawInfo := lo.Map(infos, func(info *querypb.SegmentLoadInfo, _ int) *querypb.SegmentLoadInfo {
|
||||
info = typeutil.Clone(info)
|
||||
info.IndexInfos = nil
|
||||
return info
|
||||
})
|
||||
req := &querypb.LoadSegmentsRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgID: rand.Int63(),
|
||||
TargetID: suite.node.session.ServerID,
|
||||
},
|
||||
CollectionID: suite.collectionID,
|
||||
DstNodeID: suite.node.session.ServerID,
|
||||
Infos: rawInfo,
|
||||
Schema: schema,
|
||||
NeedTransfer: false,
|
||||
LoadScope: querypb.LoadScope_Index,
|
||||
IndexInfoList: indexInfos,
|
||||
}
|
||||
|
||||
// Load segment
|
||||
status, err := suite.node.LoadSegments(ctx, req)
|
||||
suite.Require().NoError(err)
|
||||
// Ignore segment missing
|
||||
suite.Require().Equal(commonpb.ErrorCode_Success, status.GetErrorCode())
|
||||
})
|
||||
|
||||
suite.Run("loader_returns_error", func() {
|
||||
suite.TestLoadSegments_Int64()
|
||||
loader := suite.node.loader
|
||||
mockLoader := segments.NewMockLoader(suite.T())
|
||||
suite.node.loader = mockLoader
|
||||
defer func() {
|
||||
suite.node.loader = loader
|
||||
}()
|
||||
|
||||
mockLoader.EXPECT().LoadIndex(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("mocked error"))
|
||||
|
||||
indexInfos := mock_segcore.GenTestIndexInfoList(suite.collectionID, schema)
|
||||
infos := suite.genSegmentLoadInfos(schema, indexInfos)
|
||||
req := &querypb.LoadSegmentsRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgID: rand.Int63(),
|
||||
TargetID: suite.node.session.ServerID,
|
||||
},
|
||||
CollectionID: suite.collectionID,
|
||||
DstNodeID: suite.node.session.ServerID,
|
||||
Infos: infos,
|
||||
Schema: schema,
|
||||
NeedTransfer: false,
|
||||
LoadScope: querypb.LoadScope_Index,
|
||||
IndexInfoList: indexInfos,
|
||||
}
|
||||
|
||||
// Load segment
|
||||
status, err := suite.node.LoadSegments(ctx, req)
|
||||
suite.Require().NoError(err)
|
||||
suite.Require().NotEqual(commonpb.ErrorCode_Success, status.GetErrorCode())
|
||||
})
|
||||
status, err := suite.node.LoadSegments(ctx, req)
|
||||
suite.NoError(err)
|
||||
suite.ErrorIs(merr.Error(status), merr.ErrServiceInternal)
|
||||
suite.Contains(status.GetReason(), "legacy segment index load scope 2")
|
||||
suite.Contains(status.GetReason(), "use LoadScope_Reopen")
|
||||
}
|
||||
|
||||
func (suite *ServiceSuite) TestLoadSegments_Failed() {
|
||||
|
||||
@@ -447,9 +447,9 @@ message JsonStatsInfo {
|
||||
}
|
||||
|
||||
enum LoadScope {
|
||||
reserved 2;
|
||||
Full = 0;
|
||||
Delta = 1;
|
||||
Index = 2;
|
||||
Stats = 3;
|
||||
Reopen = 4;
|
||||
}
|
||||
|
||||
+502
-506
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user