furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 02:05:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance: optimize null predicate for sealed chunked fields (#50558)

Browse Source 
issue: https://github.com/milvus-io/milvus/issues/50556

https://github.com/milvus-io/milvus/pull/50436 uses BulkIsValid to
replace get_batch_views in hoping to avoid cost of constructing views
such as StringViews. However, BulkIsValid is random access where the
cost of other staff exceed the benefits which leads to performance
regression.

Constructing any kind of views is still unnecessary, and this PR uses
valid data in Chunk directly. Here benchmarks:
1 million rows of string data, `is null` expression

StringView: ~4300us
BulkIsValid: ~8340us
This PR: 1440us

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
This commit is contained in:
Spade A
2026-06-17 10:16:24 +08:00
committed by GitHub
parent 72a5447802
commit 8ad86bd133
2 changed files with 38 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/core/src/mmap/ChunkedColumnInterface.h
+37
View File
@@ -140,6 +140,43 @@ class ChunkedColumnInterface {
virtual std::vector<PinWrapper<Chunk*>>
GetAllChunks(milvus::OpContext* op_ctx) const = 0;
virtual void
ApplyValidDataInChunk(milvus::OpContext* op_ctx,
int64_t chunk_id,
int64_t offset,
int64_t size,
TargetBitmapView valid_result) const {
if (!IsNullable() || size == 0) {
return;
}
AssertInfo(offset >= 0 && size >= 0,
"Invalid valid-data range, offset: {}, size: {}",
offset,
size);
auto pw = GetChunk(op_ctx, chunk_id);
auto chunk = pw.get();
AssertInfo(offset + size <= chunk->RowNums(),
"Valid-data range out of chunk bounds, offset: {}, size: "
"{}, chunk rows: {}",
offset,
size,
chunk->RowNums());
auto& valid_data = chunk->Valid();
AssertInfo(
offset + size <= static_cast<int64_t>(valid_data.size()),
"Valid-data range out of valid-data bounds, offset: {}, size: {}, "
"valid-data size: {}",
offset,
size,
valid_data.size());
auto valid_data_ptr = valid_data.data() + offset;
for (int64_t i = 0; i < size; ++i) {
if (!valid_data_ptr[i]) {
valid_result[i] = false;
}
}
}
// Get number of rows before a specific chunk
virtual int64_t
GetNumRowsUntilChunk(int64_t chunk_id) const = 0;
internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp
+1 -31
View File
@@ -1387,37 +1387,7 @@ ChunkedSegmentSealedImpl::ApplyFieldValidData(
return;
}
auto data_type = schema_->operator[](field_id).get_data_type();
if (ChunkedColumnInterface::IsPrimitiveDataType(data_type)) {
auto pw = column->Span(op_ctx, chunk_id);
auto span = pw.get();
const bool* valid_data = span.valid_data();
if (valid_data == nullptr) {
return;
}
valid_data += offset;
for (int64_t i = 0; i < size; ++i) {
if (!valid_data[i]) {
valid_result[i] = false;
}
}
return;
}
auto row_offset = column->GetNumRowsUntilChunk(chunk_id) + offset;
std::vector<int64_t> offsets(size);
for (int64_t i = 0; i < size; ++i) {
offsets[i] = row_offset + i;
}
column->BulkIsValid(
op_ctx,
[&valid_result](bool is_valid, size_t i) {
if (!is_valid) {
valid_result[i] = false;
}
},
offsets.data(),
size);
column->ApplyValidDataInChunk(op_ctx, chunk_id, offset, size, valid_result);
}
void