mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
fix: support array length on struct array parent field (#50681)
issue: https://github.com/milvus-io/milvus/issues/50510 ref: https://github.com/milvus-io/milvus/issues/42148 --------- Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
#include "common/Array.h"
|
||||
#include "common/Json.h"
|
||||
#include "common/Tracer.h"
|
||||
#include "common/VectorArray.h"
|
||||
#include "fmt/core.h"
|
||||
#include "opentelemetry/trace/span.h"
|
||||
|
||||
@@ -119,6 +120,26 @@ PhyBinaryArithOpEvalRangeExpr::Eval(EvalCtx& context, VectorPtr& result) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case DataType::VECTOR_ARRAY: {
|
||||
auto value_type = expr_->value_.val_case();
|
||||
switch (value_type) {
|
||||
case proto::plan::GenericValue::ValCase::kInt64Val: {
|
||||
result = ExecRangeVisitorImplForVectorArray<int64_t>(input);
|
||||
break;
|
||||
}
|
||||
case proto::plan::GenericValue::ValCase::kFloatVal: {
|
||||
result = ExecRangeVisitorImplForVectorArray<double>(input);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
ThrowInfo(
|
||||
DataTypeInvalid,
|
||||
fmt::format("unsupported value type {} in expression",
|
||||
value_type));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ThrowInfo(DataTypeInvalid,
|
||||
"unsupported data type: {}",
|
||||
@@ -922,6 +943,100 @@ PhyBinaryArithOpEvalRangeExpr::ExecRangeVisitorImplForArray(
|
||||
return res_vec;
|
||||
}
|
||||
|
||||
template <typename ValueType>
|
||||
VectorPtr
|
||||
PhyBinaryArithOpEvalRangeExpr::ExecRangeVisitorImplForVectorArray(
|
||||
OffsetVector* input) {
|
||||
if (expr_->arith_op_type_ != proto::plan::ArithOpType::ArrayLength) {
|
||||
ThrowInfo(OpTypeInvalid,
|
||||
"unsupported arith type for vector array field: {}",
|
||||
expr_->arith_op_type_);
|
||||
}
|
||||
|
||||
auto real_batch_size =
|
||||
has_offset_input_ ? input->size() : GetNextBatchSize();
|
||||
if (real_batch_size == 0) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (!arg_inited_) {
|
||||
value_arg_.SetValue<ValueType>(expr_->value_);
|
||||
arg_inited_ = true;
|
||||
}
|
||||
|
||||
auto res_vec =
|
||||
std::make_shared<ColumnVector>(TargetBitmap(real_batch_size, false),
|
||||
TargetBitmap(real_batch_size, true));
|
||||
TargetBitmapView res(res_vec->GetRawData(), real_batch_size);
|
||||
TargetBitmapView valid_res(res_vec->GetValidRawData(), real_batch_size);
|
||||
|
||||
auto op_type = expr_->op_type_;
|
||||
auto value = value_arg_.GetValue<ValueType>();
|
||||
|
||||
auto compare_length = [op_type, value](int length) {
|
||||
switch (op_type) {
|
||||
case proto::plan::OpType::Equal:
|
||||
return length == value;
|
||||
case proto::plan::OpType::NotEqual:
|
||||
return length != value;
|
||||
case proto::plan::OpType::GreaterThan:
|
||||
return length > value;
|
||||
case proto::plan::OpType::GreaterEqual:
|
||||
return length >= value;
|
||||
case proto::plan::OpType::LessThan:
|
||||
return length < value;
|
||||
case proto::plan::OpType::LessEqual:
|
||||
return length <= value;
|
||||
default:
|
||||
ThrowInfo(OpTypeInvalid,
|
||||
"unsupported operator type for vector array "
|
||||
"length eval expr: {}",
|
||||
op_type);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
auto execute_sub_batch = [compare_length]<FilterType filter_type =
|
||||
FilterType::sequential>(
|
||||
const VectorArrayView* data,
|
||||
const bool* valid_data,
|
||||
const int32_t* offsets,
|
||||
const int size,
|
||||
TargetBitmapView res,
|
||||
TargetBitmapView valid_res) {
|
||||
if (data == nullptr) {
|
||||
return;
|
||||
}
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
auto offset = i;
|
||||
if constexpr (filter_type == FilterType::random) {
|
||||
offset = (offsets) ? offsets[i] : i;
|
||||
}
|
||||
if (valid_data != nullptr && !valid_data[offset]) {
|
||||
res[i] = valid_res[i] = false;
|
||||
continue;
|
||||
}
|
||||
res[i] = compare_length(data[offset].length());
|
||||
}
|
||||
};
|
||||
|
||||
int64_t processed_size = 0;
|
||||
if (has_offset_input_) {
|
||||
processed_size = ProcessDataByOffsets<VectorArrayView>(
|
||||
execute_sub_batch, std::nullptr_t{}, input, res, valid_res);
|
||||
} else {
|
||||
processed_size = ProcessDataChunks<VectorArrayView>(
|
||||
execute_sub_batch, std::nullptr_t{}, res, valid_res);
|
||||
}
|
||||
|
||||
AssertInfo(processed_size == real_batch_size,
|
||||
"internal error: expr processed rows {} not equal "
|
||||
"expect batch size {}",
|
||||
processed_size,
|
||||
real_batch_size);
|
||||
return res_vec;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
VectorPtr
|
||||
PhyBinaryArithOpEvalRangeExpr::ExecRangeVisitorImpl(OffsetVector* input) {
|
||||
|
||||
@@ -518,8 +518,9 @@ class PhyBinaryArithOpEvalRangeExpr : public SegmentExpr {
|
||||
data_type = expr_->column_.element_type_;
|
||||
}
|
||||
|
||||
// JSON and ARRAY types cannot use index for arith ops
|
||||
if (data_type == DataType::JSON || data_type == DataType::ARRAY) {
|
||||
// JSON, ARRAY and VECTOR_ARRAY types cannot use index for arith ops.
|
||||
if (data_type == DataType::JSON || data_type == DataType::ARRAY ||
|
||||
data_type == DataType::VECTOR_ARRAY) {
|
||||
exec_path_ = ExprExecPath::RawData;
|
||||
return;
|
||||
}
|
||||
@@ -592,6 +593,10 @@ class PhyBinaryArithOpEvalRangeExpr : public SegmentExpr {
|
||||
VectorPtr
|
||||
ExecRangeVisitorImplForArray(OffsetVector* input = nullptr);
|
||||
|
||||
template <typename ValueType>
|
||||
VectorPtr
|
||||
ExecRangeVisitorImplForVectorArray(OffsetVector* input = nullptr);
|
||||
|
||||
private:
|
||||
std::shared_ptr<const milvus::expr::BinaryArithOpEvalRangeExpr> expr_;
|
||||
SingleElement right_operand_arg_;
|
||||
|
||||
@@ -751,16 +751,50 @@ class SegmentExpr : public Expr {
|
||||
int64_t processed_size = 0;
|
||||
|
||||
// index reverse lookup (only for ScalarIndex path)
|
||||
if (UseIndexCursor() && num_data_chunk_ == 0) {
|
||||
return ProcessIndexLookupByOffsets<T>(
|
||||
func, skip_func, input, res, valid_res, values...);
|
||||
if constexpr (!std::is_same_v<T, VectorArrayView>) {
|
||||
if (UseIndexCursor() && num_data_chunk_ == 0) {
|
||||
return ProcessIndexLookupByOffsets<T>(
|
||||
func, skip_func, input, res, valid_res, values...);
|
||||
}
|
||||
}
|
||||
|
||||
auto& skip_index = segment_->GetSkipIndex();
|
||||
|
||||
// raw data scan
|
||||
// sealed segment
|
||||
if (segment_->type() == SegmentType::Sealed) {
|
||||
if constexpr (std::is_same_v<T, VectorArrayView>) {
|
||||
for (size_t i = 0; i < input->size(); ++i) {
|
||||
int64_t offset = (*input)[i];
|
||||
auto [chunk_id, chunk_offset] =
|
||||
segment_->get_chunk_by_offset(field_id_, offset);
|
||||
// chunk_data<VectorArrayView> would read the wrong layout:
|
||||
// storage holds VectorArray, and nullable rows may be compacted.
|
||||
// Use chunk_view to build logical VectorArrayView rows.
|
||||
auto pw = segment_->chunk_view<VectorArrayView>(
|
||||
op_ctx_,
|
||||
field_id_,
|
||||
chunk_id,
|
||||
std::make_pair(chunk_offset, int64_t{1}));
|
||||
const auto& [data_vec, valid_data] = pw.get();
|
||||
if (!skip_func || !skip_func(skip_index, field_id_, chunk_id)) {
|
||||
func.template operator()<FilterType::random>(
|
||||
data_vec.data(),
|
||||
valid_data.data(),
|
||||
nullptr,
|
||||
1,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
} else {
|
||||
ApplyValidData(valid_data.data(),
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
1);
|
||||
}
|
||||
processed_size++;
|
||||
}
|
||||
return input->size();
|
||||
} else if (segment_->type() == SegmentType::Sealed) {
|
||||
// raw data scan
|
||||
// sealed segment
|
||||
if (segment_->is_chunked()) {
|
||||
if constexpr (std::is_same_v<T, std::string_view> ||
|
||||
std::is_same_v<T, Json> ||
|
||||
@@ -1341,40 +1375,36 @@ class SegmentExpr : public Expr {
|
||||
continue; //do not go empty-loop at the bound of the chunk
|
||||
|
||||
auto& skip_index = segment_->GetSkipIndex();
|
||||
auto pw = segment_->chunk_data<T>(op_ctx_, field_id_, i);
|
||||
auto chunk = pw.get();
|
||||
const bool* valid_data = chunk.valid_data();
|
||||
if (valid_data != nullptr) {
|
||||
valid_data += data_pos;
|
||||
}
|
||||
if (!skip_func || !skip_func(skip_index, field_id_, i)) {
|
||||
const T* data = chunk.data() + data_pos;
|
||||
|
||||
if constexpr (NeedSegmentOffsets) {
|
||||
// For GIS functions: construct segment offsets array
|
||||
std::vector<int32_t> segment_offsets_array(size);
|
||||
for (int64_t j = 0; j < size; ++j) {
|
||||
segment_offsets_array[j] = static_cast<int32_t>(
|
||||
size_per_chunk_ * i + data_pos + j);
|
||||
auto process_chunk = [&](const T* data, const bool* valid_data) {
|
||||
auto skipped = skip_func && skip_func(skip_index, field_id_, i);
|
||||
if (!skipped) {
|
||||
if constexpr (NeedSegmentOffsets) {
|
||||
// For GIS functions: construct segment offsets array
|
||||
std::vector<int32_t> segment_offsets_array(size);
|
||||
for (int64_t j = 0; j < size; ++j) {
|
||||
segment_offsets_array[j] = static_cast<int32_t>(
|
||||
size_per_chunk_ * i + data_pos + j);
|
||||
}
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
segment_offsets_array.data(),
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
} else {
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
}
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
segment_offsets_array.data(),
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
} else {
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
|
||||
// Chunk is skipped by SkipIndex.
|
||||
// We still need to:
|
||||
// 1. Apply valid_data to handle nullable fields
|
||||
@@ -1384,7 +1414,6 @@ class SegmentExpr : public Expr {
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
size);
|
||||
// Call func with nullptr to update internal cursors
|
||||
if constexpr (NeedSegmentOffsets) {
|
||||
std::vector<int32_t> segment_offsets_array(size);
|
||||
for (int64_t j = 0; j < size; ++j) {
|
||||
@@ -1408,6 +1437,24 @@ class SegmentExpr : public Expr {
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
}
|
||||
};
|
||||
|
||||
if constexpr (std::is_same_v<T, VectorArrayView>) {
|
||||
// chunk_data<VectorArrayView> would read the wrong layout:
|
||||
// storage holds VectorArray, and nullable rows may be compacted.
|
||||
// Use chunk_view to build logical VectorArrayView rows.
|
||||
auto pw = segment_->chunk_view<VectorArrayView>(
|
||||
op_ctx_, field_id_, i, std::make_pair(data_pos, size));
|
||||
const auto& [data_vec, valid_data] = pw.get();
|
||||
process_chunk(data_vec.data(), valid_data.data());
|
||||
} else {
|
||||
auto pw = segment_->chunk_data<T>(op_ctx_, field_id_, i);
|
||||
auto chunk = pw.get();
|
||||
const bool* valid_data = chunk.valid_data();
|
||||
if (valid_data != nullptr) {
|
||||
valid_data += data_pos;
|
||||
}
|
||||
process_chunk(chunk.data() + data_pos, valid_data);
|
||||
}
|
||||
|
||||
processed_size += size;
|
||||
@@ -1467,7 +1514,8 @@ class SegmentExpr : public Expr {
|
||||
bool is_seal = false;
|
||||
if constexpr (std::is_same_v<T, std::string_view> ||
|
||||
std::is_same_v<T, Json> ||
|
||||
std::is_same_v<T, ArrayView>) {
|
||||
std::is_same_v<T, ArrayView> ||
|
||||
std::is_same_v<T, VectorArrayView>) {
|
||||
if (segment_->type() == SegmentType::Sealed) {
|
||||
// first is the raw data, second is valid_data
|
||||
// use valid_data to see if raw data is null
|
||||
@@ -1497,33 +1545,40 @@ class SegmentExpr : public Expr {
|
||||
is_seal = true;
|
||||
}
|
||||
}
|
||||
if (!is_seal) {
|
||||
auto pw = segment_->chunk_data<T>(op_ctx_, field_id_, i);
|
||||
auto chunk = pw.get();
|
||||
const T* data = chunk.data() + data_pos;
|
||||
const bool* valid_data = chunk.valid_data();
|
||||
if (valid_data != nullptr) {
|
||||
valid_data += data_pos;
|
||||
}
|
||||
if constexpr (std::is_same_v<T, VectorArrayView>) {
|
||||
AssertInfo(is_seal,
|
||||
"VectorArrayView must be read through chunk "
|
||||
"views");
|
||||
} else {
|
||||
if (!is_seal) {
|
||||
auto pw =
|
||||
segment_->chunk_data<T>(op_ctx_, field_id_, i);
|
||||
auto chunk = pw.get();
|
||||
const T* data = chunk.data() + data_pos;
|
||||
const bool* valid_data = chunk.valid_data();
|
||||
if (valid_data != nullptr) {
|
||||
valid_data += data_pos;
|
||||
}
|
||||
|
||||
if constexpr (NeedSegmentOffsets) {
|
||||
// For GIS functions: construct segment offsets array
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
segment_offsets_array.data(),
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
} else {
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
if constexpr (NeedSegmentOffsets) {
|
||||
// For GIS functions: construct segment offsets array
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
segment_offsets_array.data(),
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
} else {
|
||||
func(data,
|
||||
valid_data,
|
||||
nullptr,
|
||||
size,
|
||||
res + processed_size,
|
||||
valid_res + processed_size,
|
||||
values...);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1535,7 +1590,8 @@ class SegmentExpr : public Expr {
|
||||
const bool* valid_data;
|
||||
if constexpr (std::is_same_v<T, std::string_view> ||
|
||||
std::is_same_v<T, Json> ||
|
||||
std::is_same_v<T, ArrayView>) {
|
||||
std::is_same_v<T, ArrayView> ||
|
||||
std::is_same_v<T, VectorArrayView>) {
|
||||
auto pw = segment_->get_batch_views<T>(
|
||||
op_ctx_, field_id_, i, data_pos, size);
|
||||
valid_data = pw.get().second.data();
|
||||
|
||||
@@ -769,6 +769,142 @@ TEST(Expr, TestVectorArrayNullExpr) {
|
||||
(void)fakevec_fid;
|
||||
}
|
||||
|
||||
TEST(Expr, TestVectorArrayLengthExpr) {
|
||||
auto schema = std::make_shared<Schema>();
|
||||
auto fakevec_fid = schema->AddDebugField(
|
||||
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
|
||||
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
|
||||
auto vector_array_fid =
|
||||
schema->AddDebugVectorArrayField("array_float_vector",
|
||||
DataType::VECTOR_FLOAT,
|
||||
4,
|
||||
knowhere::metric::L2,
|
||||
true);
|
||||
schema->set_primary_field_id(i64_fid);
|
||||
|
||||
constexpr int N = 128;
|
||||
constexpr int kArrayLen = 2;
|
||||
auto raw_data = DataGen(schema, N, 42, 0, 1, kArrayLen);
|
||||
auto valid_data = raw_data.get_col_valid(vector_array_fid);
|
||||
auto vector_array_col =
|
||||
raw_data.get_col<VectorFieldProto>(vector_array_fid);
|
||||
|
||||
auto growing = CreateGrowingSegment(schema, empty_index_meta);
|
||||
auto offset = growing->PreInsert(N);
|
||||
growing->Insert(offset,
|
||||
N,
|
||||
raw_data.row_ids_.data(),
|
||||
raw_data.timestamps_.data(),
|
||||
raw_data.raw_);
|
||||
auto growing_segment = dynamic_cast<SegmentGrowingImpl*>(growing.get());
|
||||
ASSERT_NE(growing_segment, nullptr);
|
||||
|
||||
std::vector<uint8_t> valid_bitmap((N + 7) / 8, 0);
|
||||
std::vector<milvus::VectorArray> vector_arrays;
|
||||
vector_arrays.reserve(N);
|
||||
for (int i = 0; i < N; ++i) {
|
||||
if (valid_data[i]) {
|
||||
valid_bitmap[i >> 3] |= 1 << (i & 0x07);
|
||||
vector_arrays.emplace_back(vector_array_col[i]);
|
||||
}
|
||||
}
|
||||
|
||||
auto field_data = storage::CreateFieldData(
|
||||
DataType::VECTOR_ARRAY, DataType::VECTOR_FLOAT, true, 4);
|
||||
field_data->FillFieldData(vector_arrays.data(), valid_bitmap.data(), N, 0);
|
||||
|
||||
auto cm = storage::RemoteChunkManagerSingleton::GetInstance()
|
||||
.GetRemoteChunkManager();
|
||||
auto sealed_segment = CreateSealedSegment(schema);
|
||||
auto field_data_info =
|
||||
PrepareSingleFieldInsertBinlog(kCollectionID,
|
||||
kPartitionID,
|
||||
kSegmentID,
|
||||
vector_array_fid.get(),
|
||||
{field_data},
|
||||
cm);
|
||||
sealed_segment->LoadFieldData(field_data_info);
|
||||
|
||||
auto make_plan = [&](proto::plan::OpType op, int64_t target) {
|
||||
proto::plan::GenericValue value;
|
||||
value.set_int64_val(target);
|
||||
proto::plan::GenericValue right_operand;
|
||||
right_operand.set_int64_val(0);
|
||||
auto expr = std::make_shared<milvus::expr::BinaryArithOpEvalRangeExpr>(
|
||||
milvus::expr::ColumnInfo(vector_array_fid,
|
||||
DataType::VECTOR_ARRAY,
|
||||
DataType::VECTOR_FLOAT,
|
||||
{},
|
||||
true),
|
||||
op,
|
||||
proto::plan::ArithOpType::ArrayLength,
|
||||
value,
|
||||
right_operand);
|
||||
return std::make_shared<plan::FilterBitsNode>(DEFAULT_PLANNODE_ID,
|
||||
expr);
|
||||
};
|
||||
|
||||
struct Testcase {
|
||||
proto::plan::OpType op;
|
||||
int64_t target;
|
||||
std::function<bool(bool)> ref_func;
|
||||
};
|
||||
std::vector<Testcase> testcases = {
|
||||
{proto::plan::OpType::Equal,
|
||||
kArrayLen,
|
||||
[](bool valid) { return valid; }},
|
||||
{proto::plan::OpType::NotEqual,
|
||||
kArrayLen + 1,
|
||||
[](bool valid) { return valid; }},
|
||||
{proto::plan::OpType::GreaterThan,
|
||||
kArrayLen - 1,
|
||||
[](bool valid) { return valid; }},
|
||||
{proto::plan::OpType::GreaterEqual,
|
||||
kArrayLen,
|
||||
[](bool valid) { return valid; }},
|
||||
{proto::plan::OpType::LessThan,
|
||||
kArrayLen,
|
||||
[](bool valid) { return false; }},
|
||||
{proto::plan::OpType::LessEqual,
|
||||
kArrayLen,
|
||||
[](bool valid) { return valid; }},
|
||||
};
|
||||
|
||||
for (const auto& testcase : testcases) {
|
||||
std::array<const SegmentInternalInterface*, 2> segments = {
|
||||
static_cast<const SegmentInternalInterface*>(growing_segment),
|
||||
static_cast<const SegmentInternalInterface*>(sealed_segment.get())};
|
||||
for (auto* segment : segments) {
|
||||
auto plan = make_plan(testcase.op, testcase.target);
|
||||
auto final = ExecuteQueryExpr(plan, segment, N, MAX_TIMESTAMP);
|
||||
EXPECT_EQ(final.size(), N);
|
||||
for (int i = 0; i < N; ++i) {
|
||||
ASSERT_EQ(final[i], testcase.ref_func(valid_data[i]))
|
||||
<< "segment type " << segment->type() << ", row " << i;
|
||||
}
|
||||
|
||||
milvus::exec::OffsetVector offsets;
|
||||
offsets.reserve(N / 2);
|
||||
for (int i = 0; i < N; ++i) {
|
||||
if (i % 2 == 0) {
|
||||
offsets.emplace_back(i);
|
||||
}
|
||||
}
|
||||
auto col_vec = milvus::test::gen_filter_res(
|
||||
plan.get(), segment, N, MAX_TIMESTAMP, &offsets);
|
||||
BitsetTypeView view(col_vec->GetRawData(), col_vec->size());
|
||||
ASSERT_EQ(view.size(), offsets.size());
|
||||
for (int i = 0; i < offsets.size(); ++i) {
|
||||
ASSERT_EQ(view[i], testcase.ref_func(valid_data[offsets[i]]))
|
||||
<< "segment type " << segment->type() << ", offset row "
|
||||
<< offsets[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(void)fakevec_fid;
|
||||
}
|
||||
|
||||
TEST(Expr, PraseArrayContainsExpr) {
|
||||
// Test expressions for array_contains operations
|
||||
std::vector<const char*> exprs{
|
||||
|
||||
@@ -1284,8 +1284,90 @@ SegmentGrowingImpl::chunk_vector_array_view_impl(
|
||||
FieldId field_id,
|
||||
int64_t chunk_id,
|
||||
std::optional<std::pair<int64_t, int64_t>> offset_len) const {
|
||||
ThrowInfo(ErrorCode::NotImplemented,
|
||||
"chunk vector array view impl not implement for growing segment");
|
||||
(void)op_ctx;
|
||||
|
||||
auto& field_meta = schema_->operator[](field_id);
|
||||
AssertInfo(field_meta.get_data_type() == DataType::VECTOR_ARRAY,
|
||||
"chunk_vector_array_view_impl only supports VECTOR_ARRAY field");
|
||||
|
||||
auto vector_data = insert_record_.get_data<VectorArray>(field_id);
|
||||
auto size_per_chunk = vector_data->get_size_per_chunk();
|
||||
auto active_count = insert_record_.ack_responder_.GetAck();
|
||||
auto start_offset = int64_t{0};
|
||||
auto len = size_per_chunk;
|
||||
if (offset_len.has_value()) {
|
||||
start_offset = offset_len->first;
|
||||
len = offset_len->second;
|
||||
}
|
||||
|
||||
AssertInfo(start_offset >= 0 && start_offset < size_per_chunk,
|
||||
"Retrieve vector array views with out-of-bound offset:{}, "
|
||||
"len:{}, wrong",
|
||||
start_offset,
|
||||
len);
|
||||
AssertInfo(len >= 0 && len <= size_per_chunk,
|
||||
"Retrieve vector array views with out-of-bound offset:{}, "
|
||||
"len:{}, wrong",
|
||||
start_offset,
|
||||
len);
|
||||
AssertInfo(start_offset + len <= size_per_chunk,
|
||||
"Retrieve vector array views with out-of-bound offset:{}, "
|
||||
"len:{}, wrong",
|
||||
start_offset,
|
||||
len);
|
||||
|
||||
auto logical_start = chunk_id * size_per_chunk + start_offset;
|
||||
AssertInfo(logical_start >= 0 && logical_start <= active_count,
|
||||
"Retrieve vector array views with out-of-bound offset:{}, "
|
||||
"len:{}, wrong",
|
||||
start_offset,
|
||||
len);
|
||||
if (offset_len.has_value()) {
|
||||
AssertInfo(logical_start + len <= active_count,
|
||||
"Retrieve vector array views with out-of-bound offset:{}, "
|
||||
"len:{}, wrong",
|
||||
start_offset,
|
||||
len);
|
||||
} else {
|
||||
len = std::min(len, active_count - logical_start);
|
||||
}
|
||||
|
||||
std::vector<VectorArrayView> views;
|
||||
views.reserve(len);
|
||||
FixedVector<bool> valid_data;
|
||||
ThreadSafeValidDataPtr valid_vec_ptr = nullptr;
|
||||
if (field_meta.is_nullable()) {
|
||||
valid_vec_ptr = insert_record_.get_valid_data(field_id);
|
||||
valid_data.reserve(len);
|
||||
}
|
||||
|
||||
for (int64_t i = 0; i < len; ++i) {
|
||||
auto logical_offset = logical_start + i;
|
||||
if (field_meta.is_nullable()) {
|
||||
auto valid = valid_vec_ptr->is_valid(logical_offset);
|
||||
valid_data.push_back(valid);
|
||||
if (!valid) {
|
||||
views.emplace_back();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
auto vector_array = vector_data->get_element(logical_offset);
|
||||
AssertInfo(vector_array != nullptr,
|
||||
"Cannot find VECTOR_ARRAY data at segment offset {}",
|
||||
logical_offset);
|
||||
views.emplace_back(const_cast<char*>(vector_array->data()),
|
||||
vector_array->dim(),
|
||||
vector_array->length(),
|
||||
vector_array->byte_size(),
|
||||
vector_array->get_element_type());
|
||||
}
|
||||
|
||||
std::pair<std::vector<VectorArrayView>, FixedVector<bool>> content{
|
||||
std::move(views), std::move(valid_data)};
|
||||
return PinWrapper<
|
||||
std::pair<std::vector<VectorArrayView>, FixedVector<bool>>>(
|
||||
std::move(content));
|
||||
}
|
||||
|
||||
PinWrapper<std::pair<std::vector<std::string_view>, FixedVector<bool>>>
|
||||
|
||||
@@ -2281,17 +2281,28 @@ func (v *ParserVisitor) VisitArrayLength(ctx *parser.ArrayLengthContext) interfa
|
||||
FieldId: field.FieldID,
|
||||
DataType: field.DataType,
|
||||
ElementType: field.GetElementType(),
|
||||
Nullable: field.GetNullable(),
|
||||
}
|
||||
} else {
|
||||
columnInfo, err = v.getChildColumnInfo(ctx.Identifier(), ctx.JSONIdentifier(), nil, nil)
|
||||
if ctx.Identifier() != nil {
|
||||
if parentColumnInfo, ok, parentErr := v.getStructArrayParentColumnInfo(ctx.Identifier().GetText()); ok || parentErr != nil {
|
||||
columnInfo = parentColumnInfo
|
||||
err = parentErr
|
||||
}
|
||||
}
|
||||
if columnInfo == nil && err == nil {
|
||||
columnInfo, err = v.getChildColumnInfo(ctx.Identifier(), ctx.JSONIdentifier(), nil, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if columnInfo == nil ||
|
||||
(!typeutil.IsJSONType(columnInfo.GetDataType()) && !typeutil.IsArrayType(columnInfo.GetDataType())) {
|
||||
(!typeutil.IsJSONType(columnInfo.GetDataType()) &&
|
||||
!typeutil.IsArrayType(columnInfo.GetDataType()) &&
|
||||
!typeutil.IsVectorArrayType(columnInfo.GetDataType())) {
|
||||
return merr.WrapErrParameterInvalidMsg(
|
||||
"array_length operation are only supported on json or array fields now, got: %s", ctx.GetText())
|
||||
"array_length operation are only supported on json, array or array-of-vector fields now, got: %s", ctx.GetText())
|
||||
}
|
||||
|
||||
expr := &planpb.Expr{
|
||||
|
||||
@@ -990,6 +990,7 @@ func TestExpr_VectorArrayOnlyStructParentIsNull(t *testing.T) {
|
||||
Name: "vector_struct[embeddings]",
|
||||
DataType: schemapb.DataType_ArrayOfVector,
|
||||
ElementType: schemapb.DataType_FloatVector,
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "4"},
|
||||
},
|
||||
@@ -2149,6 +2150,7 @@ func Test_ArrayLength(t *testing.T) {
|
||||
`array_length(StringArrayField) > 5`,
|
||||
`array_length(StringArrayField) >= 5`,
|
||||
// struct array sub-field
|
||||
`array_length(struct_array) == 2`,
|
||||
`array_length(struct_array[sub_str]) == 3`,
|
||||
`array_length(struct_array[sub_int]) > 1`,
|
||||
`array_length(struct_array[sub_int]) <= 10`,
|
||||
@@ -3188,7 +3190,7 @@ func TestExpr_StructIndexField_PlanShape(t *testing.T) {
|
||||
func TestExpr_StructFieldArrayLength(t *testing.T) {
|
||||
schema := newTestSchemaHelper(t)
|
||||
|
||||
ret := handleExpr(schema, `array_length(struct_array[sub_int])`)
|
||||
ret := handleExpr(schema, `array_length(struct_array)`)
|
||||
ewt, ok := ret.(*ExprWithType)
|
||||
require.True(t, ok, "handleExpr should return *ExprWithType")
|
||||
assert.Equal(t, schemapb.DataType_Int64, ewt.dataType)
|
||||
@@ -3200,12 +3202,30 @@ func TestExpr_StructFieldArrayLength(t *testing.T) {
|
||||
|
||||
columnInfo := bae.GetLeft().GetColumnExpr().GetInfo()
|
||||
require.NotNil(t, columnInfo)
|
||||
assert.Equal(t, int64(133), columnInfo.GetFieldId())
|
||||
assert.Equal(t, schemapb.DataType_Array, columnInfo.GetDataType())
|
||||
assert.Equal(t, schemapb.DataType_VarChar, columnInfo.GetElementType())
|
||||
assert.Empty(t, columnInfo.GetNestedPath())
|
||||
|
||||
ret = handleExpr(schema, `array_length(struct_array[sub_int])`)
|
||||
ewt, ok = ret.(*ExprWithType)
|
||||
require.True(t, ok, "handleExpr should return *ExprWithType")
|
||||
assert.Equal(t, schemapb.DataType_Int64, ewt.dataType)
|
||||
|
||||
bae = ewt.expr.GetBinaryArithExpr()
|
||||
require.NotNil(t, bae)
|
||||
assert.Equal(t, planpb.ArithOpType_ArrayLength, bae.GetOp())
|
||||
require.Nil(t, bae.GetRight())
|
||||
|
||||
columnInfo = bae.GetLeft().GetColumnExpr().GetInfo()
|
||||
require.NotNil(t, columnInfo)
|
||||
assert.Equal(t, int64(134), columnInfo.GetFieldId())
|
||||
assert.Equal(t, schemapb.DataType_Array, columnInfo.GetDataType())
|
||||
assert.Equal(t, schemapb.DataType_Int32, columnInfo.GetElementType())
|
||||
assert.Empty(t, columnInfo.GetNestedPath())
|
||||
|
||||
validExprs := []string{
|
||||
`array_length(struct_array) == 10`,
|
||||
`array_length(struct_array[sub_int]) == 10`,
|
||||
`array_length(struct_array[sub_str]) > 0`,
|
||||
}
|
||||
@@ -3220,6 +3240,58 @@ func TestExpr_StructFieldArrayLength(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpr_VectorArrayOnlyStructParentArrayLength(t *testing.T) {
|
||||
schema := newTestSchema(false)
|
||||
schema.StructArrayFields = append(schema.StructArrayFields, &schemapb.StructArrayFieldSchema{
|
||||
FieldID: 10000,
|
||||
Name: "vector_struct",
|
||||
Nullable: true,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{
|
||||
FieldID: 10001,
|
||||
Name: "vector_struct[embeddings]",
|
||||
DataType: schemapb.DataType_ArrayOfVector,
|
||||
ElementType: schemapb.DataType_FloatVector,
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "4"},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
helper, err := typeutil.CreateSchemaHelper(schema)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, expr := range []string{
|
||||
`array_length(vector_struct)`,
|
||||
`array_length(vector_struct[embeddings])`,
|
||||
} {
|
||||
ret := handleExpr(helper, expr)
|
||||
ewt, ok := ret.(*ExprWithType)
|
||||
require.True(t, ok, "handleExpr should return *ExprWithType")
|
||||
assert.Equal(t, schemapb.DataType_Int64, ewt.dataType)
|
||||
|
||||
bae := ewt.expr.GetBinaryArithExpr()
|
||||
require.NotNil(t, bae)
|
||||
assert.Equal(t, planpb.ArithOpType_ArrayLength, bae.GetOp())
|
||||
|
||||
columnInfo := bae.GetLeft().GetColumnExpr().GetInfo()
|
||||
require.NotNil(t, columnInfo)
|
||||
assert.Equal(t, int64(10001), columnInfo.GetFieldId())
|
||||
assert.Equal(t, schemapb.DataType_ArrayOfVector, columnInfo.GetDataType())
|
||||
assert.Equal(t, schemapb.DataType_FloatVector, columnInfo.GetElementType())
|
||||
assert.True(t, columnInfo.GetNullable())
|
||||
}
|
||||
|
||||
_, err = CreateSearchPlan(helper, `array_length(vector_struct) == 2`, "FloatVectorField", &planpb.QueryInfo{
|
||||
Topk: 0,
|
||||
MetricType: "",
|
||||
SearchParams: "",
|
||||
RoundDecimal: 0,
|
||||
}, nil, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestExpr_StructIndexField_RangeForms(t *testing.T) {
|
||||
schema := newTestSchemaHelper(t)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user