From 9cef600b7d9984d324ffe3378de0e01917995ff1 Mon Sep 17 00:00:00 2001 From: Spade A <71589810+SpadeA-Tang@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:50:40 +0800 Subject: [PATCH] fix: correct nullable vector array row IDs in sealed bf search (#51508) issue: https://github.com/milvus-io/milvus/issues/51414 Signed-off-by: SpadeA --- internal/core/src/query/SearchOnSealed.cpp | 14 +- .../src/segcore/ChunkedSegmentSealedTest.cpp | 127 ++++++++++++++++++ 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/internal/core/src/query/SearchOnSealed.cpp b/internal/core/src/query/SearchOnSealed.cpp index 0399345c3c..5db902d6f7 100644 --- a/internal/core/src/query/SearchOnSealed.cpp +++ b/internal/core/src/query/SearchOnSealed.cpp @@ -176,7 +176,13 @@ SearchOnSealedColumn(const Schema& schema, CheckBruteForceSearchParam(field, search_info); - if (column->IsNullable()) { + // Nullable plain-vector chunks compact away NULL rows and therefore need + // physical/logical row mapping. VECTOR_ARRAY raw chunks keep one list + // offset per logical row; NULL and empty rows are represented by + // zero-length lists, so row mapping must not be built or applied here. + const bool needs_offset_mapping = + column->IsNullable() && data_type != DataType::VECTOR_ARRAY; + if (needs_offset_mapping) { column->BuildValidRowIds(op_context); } @@ -192,7 +198,7 @@ SearchOnSealedColumn(const Schema& schema, TargetBitmap transformed_bitset; BitsetView search_bitview = bitview; const auto has_offset_mapping = - offset_mapping.IsEnabled() && !is_element_level_search; + needs_offset_mapping && offset_mapping.IsEnabled(); if (has_offset_mapping) { if (offset_mapping.GetValidCount() == 0) { // All vectors are null, return empty result @@ -278,6 +284,10 @@ SearchOnSealedColumn(const Schema& schema, offsets_pw = column->VectorArrayOffsets(op_context, i); raw_dataset.raw_data_offsets = offsets_pw.get(); + if (raw_dataset.raw_data_offsets[chunk_size] == 0) { + offset += chunk_size; + continue; + } } if (use_vector_iterator) { diff --git a/internal/core/src/segcore/ChunkedSegmentSealedTest.cpp b/internal/core/src/segcore/ChunkedSegmentSealedTest.cpp index 18d859c95d..7ce0aeb625 100644 --- a/internal/core/src/segcore/ChunkedSegmentSealedTest.cpp +++ b/internal/core/src/segcore/ChunkedSegmentSealedTest.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -562,6 +563,132 @@ TEST(test_chunk_segment, SearchOnSealedColumnBruteForceUsesOriginalTopk) { ASSERT_EQ(search_result.distances_.size(), search_info.topk_); } +TEST(test_chunk_segment, + SearchOnSealedColumnNullableVectorArrayUsesLogicalRowOffsets) { + constexpr int64_t dim = 2; + constexpr int64_t row_count = 6; + constexpr int64_t vector_count = 3; + + auto schema = std::make_shared(); + auto vec_id = + schema->AddDebugVectorArrayField("profile[embedding]", + DataType::VECTOR_FLOAT, + dim, + knowhere::metric::MAX_SIM_COSINE, + true); + auto field_meta = schema->operator[](vec_id); + + const auto bitmap_bytes = (row_count + 7) / 8; + const auto header_bytes = sizeof(uint32_t) * (row_count * 2 + 1); + const auto payload_offset = + static_cast(bitmap_bytes + header_bytes); + const auto vector_bytes = static_cast(dim * sizeof(float)); + const std::array payload{ + 1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F}; + std::vector buffer(bitmap_bytes + header_bytes + + payload.size() * sizeof(float) + + MMAP_ARRAY_PADDING, + 0); + + // Logical rows: [NULL, [], [A], [B], [], [C]]. + buffer[0] = 0b00111110; + const std::array header{ + payload_offset, + 0, + payload_offset, + 0, + payload_offset, + 1, + payload_offset + vector_bytes, + 1, + payload_offset + 2 * vector_bytes, + 0, + payload_offset + 2 * vector_bytes, + 1, + payload_offset + 3 * vector_bytes}; + memcpy(buffer.data() + bitmap_bytes, + header.data(), + header.size() * sizeof(uint32_t)); + memcpy(buffer.data() + payload_offset, + payload.data(), + payload.size() * sizeof(float)); + + std::vector> chunks; + auto chunk_mmap_guard = std::make_shared(nullptr, 0, ""); + chunks.emplace_back( + std::make_unique(dim, + row_count, + buffer.data(), + buffer.size(), + DataType::VECTOR_FLOAT, + chunk_mmap_guard, + true)); + auto translator = std::make_unique( + std::vector{row_count}, "", std::move(chunks)); + auto slot = + cachinglayer::Manager::GetInstance().CreateCacheSlot( + std::move(translator), nullptr); + auto column = MakeChunkedColumnBase( + DataType::VECTOR_ARRAY, std::move(slot), field_meta); + ASSERT_FALSE(column->GetOffsetMapping().IsEnabled()); + auto offsets_pw = column->VectorArrayOffsets(nullptr, 0); + const std::array expected_offsets{ + 0, 0, 0, 1, 2, 2, 3}; + for (int64_t i = 0; i <= row_count; ++i) { + ASSERT_EQ(offsets_pw.get()[i], expected_offsets[i]); + } + + SearchInfo search_info; + search_info.search_params_ = knowhere::Json{ + {knowhere::meta::METRIC_TYPE, knowhere::metric::MAX_SIM_COSINE}}; + search_info.field_id_ = vec_id; + search_info.metric_type_ = knowhere::metric::MAX_SIM_COSINE; + search_info.topk_ = 3; + + const std::array query{1.0F, 0.0F}; + const std::array query_offsets{0, 1}; + const std::map index_info; + milvus::OpContext op_context; + + auto run_search = [&](const BitsetView& bitset) { + SearchResult result; + query::SearchOnSealedColumn(*schema, + column.get(), + search_info, + index_info, + query.data(), + query_offsets.data(), + 1, + row_count, + bitset, + &op_context, + result); + return result; + }; + + auto result = run_search(BitsetView{}); + ASSERT_EQ(result.seg_offsets_.size(), 3); + ASSERT_EQ(result.distances_.size(), 3); + EXPECT_EQ(result.seg_offsets_[0], 2); + EXPECT_EQ(result.seg_offsets_[1], 3); + EXPECT_EQ(result.seg_offsets_[2], 5); + EXPECT_FLOAT_EQ(result.distances_[0], 1.0F); + EXPECT_FLOAT_EQ(result.distances_[1], 0.0F); + EXPECT_FLOAT_EQ(result.distances_[2], -1.0F); + + // Bitsets are also in logical row space for sealed raw VECTOR_ARRAY. + std::array filtered_bits{1U << 3}; + auto filtered_result = + run_search(BitsetView(filtered_bits.data(), row_count)); + ASSERT_EQ(filtered_result.seg_offsets_.size(), 3); + EXPECT_EQ(filtered_result.seg_offsets_[0], 2); + EXPECT_EQ(filtered_result.seg_offsets_[1], 5); + EXPECT_EQ(filtered_result.seg_offsets_[2], INVALID_SEG_OFFSET); + EXPECT_FLOAT_EQ(filtered_result.distances_[0], 1.0F); + EXPECT_FLOAT_EQ(filtered_result.distances_[1], -1.0F); + EXPECT_FALSE(column->GetOffsetMapping().IsEnabled()); +} + // Test search on nullable vector field with all null vectors // This test verifies that SearchOnSealedColumn returns empty results // instead of crashing when all vectors are null