fix: [cp2.6]preserve null semantics in filter execution (#50722)

issue: #50698
pr: #50701

Fixes #50698

## What this PR does

This is the 2.6 backport of #50701. It fixes NULL/UNKNOWN validity
handling in boolean filter execution paths:

- Makes conjunctive filter short-circuit decisions preserve rows that
are still UNKNOWN and may be resolved by later predicates.
- Treats UNKNOWN predicate results as filtered out when converting
predicate results to filter bitsets.
- Preserves nullable parent JSON validity for `!=` instead of producing
`data=true, valid=false` that can leak through raw-bit consumers.
- Fixes chunked sealed variable-length offset filtering so skipped NULL
rows keep invalid validity instead of becoming definite false.

## Verification

Validation reported for this 2.6 cherry-pick:

- `git diff --check` passed.
- `go test -buildvcs=false -count=1
./internal/parser/planparserv2/rewriter` passed.

C++ validation was blocked in the local environment because
`cmake_build` attempted to regenerate with Conan-1-style
`conanbuildinfo.cmake`, while local Conan output was Conan 2 and did not
include that file; syntax-only compile also hit missing `bsoncxx`
headers in the local dependency cache.

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>

internal/core/src/exec/expression/ConjunctExpr.cpp

@@ -47,14 +47,23 @@ PhyConjunctFilterExpr::UpdateResult(ColumnVectorPtr& input_result,
         common::ThreeValuedLogicOp::Or(result, input_result);
     }
 
-    // Return the count of active rows for short-circuit optimization
-    // For AND: return count of true (if 0, all false, can skip)
-    // For OR: return count of false (if 0, all true, can skip)
-    TargetBitmapView res_data(result->GetRawData(), result->size());
+    // Return rows that still need the following expressions.
+    // For AND: TRUE or NULL rows still need evaluation; only definite FALSE can
+    // stop. For OR: FALSE or NULL rows still need evaluation; only definite TRUE
+    // can stop.
+    const size_t size = result->size();
+    TargetBitmapView res_data(result->GetRawData(), size);
+    TargetBitmapView res_valid(result->GetValidRawData(), size);
     if (is_and_) {
-        return static_cast<int64_t>(res_data.count());
+        TargetBitmap active_rows(res_valid);
+        active_rows.inplace_sub(res_data, size);  // valid & ~data
+        active_rows.flip();                       // data | ~valid
+        return static_cast<int64_t>(active_rows.count());
     } else {
-        return static_cast<int64_t>(result->size() - res_data.count());
+        TargetBitmap active_rows(res_data);
+        active_rows.inplace_and(res_valid, size);  // data & valid
+        active_rows.flip();                        // ~data | ~valid
+        return static_cast<int64_t>(active_rows.count());
     }
 }
 

internal/core/src/exec/expression/ConjunctExprTest.cpp

@@ -0,0 +1,112 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/Types.h"
+#include "common/Vector.h"
+#include "exec/QueryContext.h"
+#include "exec/expression/ConjunctExpr.h"
+#include "exec/expression/EvalCtx.h"
+
+namespace milvus::exec {
+namespace {
+
+class FixedBitmapExpr : public Expr {
+ public:
+    FixedBitmapExpr(TargetBitmap data, TargetBitmap valid)
+        : Expr(DataType::BOOL, {}, "FixedBitmapExpr", nullptr),
+          data_(std::move(data)),
+          valid_(std::move(valid)) {
+    }
+
+    void
+    Eval(EvalCtx&, VectorPtr& result) override {
+        ++eval_count_;
+        result = std::make_shared<ColumnVector>(data_.clone(), valid_.clone());
+    }
+
+    void
+    MoveCursor() override {
+        ++move_count_;
+    }
+
+    std::string
+    ToString() const override {
+        return "FixedBitmapExpr";
+    }
+
+    std::optional<milvus::expr::ColumnInfo>
+    GetColumnInfo() const override {
+        return std::nullopt;
+    }
+
+    int eval_count_ = 0;
+    int move_count_ = 0;
+
+ private:
+    TargetBitmap data_;
+    TargetBitmap valid_;
+};
+
+std::shared_ptr<FixedBitmapExpr>
+FixedBool(const bool data, const bool valid) {
+    TargetBitmap data_bitmap(1, data);
+    TargetBitmap valid_bitmap(1, valid);
+    return std::make_shared<FixedBitmapExpr>(std::move(data_bitmap),
+                                             std::move(valid_bitmap));
+}
+
+}  // namespace
+
+TEST(ConjunctExprTest, AndKeepsUnknownRowsActiveForFollowingFalse) {
+    auto unknown = FixedBool(false, false);
+    auto true_expr = FixedBool(true, true);
+    auto false_expr = FixedBool(false, true);
+
+    std::vector<ExprPtr> inputs{unknown, true_expr, false_expr};
+    PhyConjunctFilterExpr conjunct(std::move(inputs), true, nullptr);
+
+    QueryContext query_context("conjunct_test", nullptr, 1, 0);
+    ExecContext exec_context(&query_context);
+    EvalCtx eval_context(&exec_context);
+
+    VectorPtr result;
+    conjunct.Eval(eval_context, result);
+
+    auto output = std::dynamic_pointer_cast<ColumnVector>(result);
+    ASSERT_NE(output, nullptr);
+    ASSERT_TRUE(output->IsBitmap());
+
+    TargetBitmapView data(output->GetRawData(), output->size());
+    TargetBitmapView valid(output->GetValidRawData(), output->size());
+    ASSERT_EQ(output->size(), 1);
+    EXPECT_FALSE(data[0]);
+    EXPECT_TRUE(valid[0]);
+
+    EXPECT_EQ(unknown->eval_count_, 1);
+    EXPECT_EQ(true_expr->eval_count_, 1);
+    EXPECT_EQ(false_expr->eval_count_, 1);
+    EXPECT_EQ(false_expr->move_count_, 0);
+}
+
+}  // namespace milvus::exec

internal/core/src/exec/expression/Expr.h

@@ -557,8 +557,7 @@ class SegmentExpr : public Expr {
                                 valid_res + processed_size,
                                 values...);
                         } else {
-                            if (valid_data.size() > processed_size &&
-                                !valid_data[processed_size]) {
+                            if (!valid_data.empty() && !valid_data[0]) {
                                 res[processed_size] =
                                     valid_res[processed_size] = false;
                             }

internal/core/src/exec/expression/ExprJsonAdvancedTest.cpp

@@ -239,7 +239,7 @@ TEST_P(ExprTest, TestUnaryRangeWithJSONNullable) {
              [](std::variant<int64_t, bool, double, std::string_view> v,
                 bool valid) {
                  if (!valid) {
-                     return true;
+                     return false;
                  }
                  return !std::get<bool>(v);
              },

internal/core/src/exec/expression/ExprJsonRangeTest.cpp

@@ -1002,7 +1002,7 @@ TEST_P(ExprTest, TestUnaryRangeJsonNullable) {
                 case OpType::NotEqual: {
                     f = [&](int64_t value, bool valid) {
                         if (!valid) {
-                            return true;
+                            return false;
                         }
                         return value != testcase.val;
                     };
@@ -1132,7 +1132,7 @@ TEST_P(ExprTest, TestUnaryRangeJsonNullable) {
     for (const auto& testcase : array_cases) {
         auto check = [&](OpType op, bool valid) {
             if (!valid) {
-                return op == OpType::NotEqual ? true : false;
+                return false;
             }
             if (testcase.nested_path[0] == "array" && op == OpType::Equal) {
                 return true;

internal/core/src/exec/expression/JsonContainsByStatsTest.cpp

@@ -9,6 +9,8 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License
 
+#include <arrow/builder.h>
+#include <fmt/core.h>
 #include <gtest/gtest.h>
 #include <cstdint>
 #include <memory>
@@ -17,6 +19,7 @@
 
 #include "common/Schema.h"
 #include "common/Types.h"
+#include "exec/expression/UnaryExpr.h"
 #include "expr/ITypeExpr.h"
 #include "index/json_stats/JsonKeyStats.h"
 #include "cachinglayer/Manager.h"
@@ -37,6 +40,31 @@ using namespace milvus::index;
 
 namespace {
 
+bool
+IsValidAt(const std::vector<uint8_t>& valid_data, size_t i) {
+    return ((valid_data[i >> 3] >> (i & 0x07)) & 1) != 0;
+}
+
+std::shared_ptr<arrow::BinaryArray>
+MakeNullableJsonArray(const std::vector<std::string>& json_strings,
+                      const std::vector<uint8_t>& valid_data) {
+    arrow::BinaryBuilder builder;
+    for (size_t i = 0; i < json_strings.size(); ++i) {
+        auto status = IsValidAt(valid_data, i) ? builder.Append(json_strings[i])
+                                               : builder.AppendNull();
+        AssertInfo(status.ok(),
+                   "failed to build nullable JSON Arrow array: {}",
+                   status.ToString());
+    }
+
+    std::shared_ptr<arrow::Array> array;
+    auto status = builder.Finish(&array);
+    AssertInfo(status.ok(),
+               "failed to finish nullable JSON Arrow array: {}",
+               status.ToString());
+    return std::static_pointer_cast<arrow::BinaryArray>(array);
+}
+
 milvus::index::CacheJsonKeyStatsPtr
 BuildAndLoadJsonKeyStats(const std::vector<std::string>& json_strings,
                          const milvus::FieldId json_fid,
@@ -46,16 +74,23 @@ BuildAndLoadJsonKeyStats(const std::vector<std::string>& json_strings,
                          int64_t segment_id,
                          int64_t field_id,
                          int64_t build_id,
-                         int64_t version_id) {
+                         int64_t version_id,
+                         const std::vector<uint8_t>* valid_data = nullptr) {
     std::vector<milvus::Json> data;
     data.reserve(json_strings.size());
     for (const auto& s : json_strings) {
         data.emplace_back(simdjson::padded_string(s));
     }
 
+    auto nullable = valid_data != nullptr;
     auto field_data =
-        std::make_shared<FieldData<milvus::Json>>(DataType::JSON, false);
-    field_data->add_json_data(data);
+        std::make_shared<FieldData<milvus::Json>>(DataType::JSON, nullable);
+    if (valid_data != nullptr) {
+        field_data->FillFieldData(
+            MakeNullableJsonArray(json_strings, *valid_data));
+    } else {
+        field_data->add_json_data(data);
+    }
 
     auto payload_reader =
         std::make_shared<milvus::storage::PayloadReader>(field_data);
@@ -64,6 +99,7 @@ BuildAndLoadJsonKeyStats(const std::vector<std::string>& json_strings,
     proto::schema::FieldSchema field_schema;
     field_schema.set_data_type(proto::schema::DataType::JSON);
     field_schema.set_fieldid(json_fid.get());
+    field_schema.set_nullable(nullable);
 
     storage::FieldDataMeta field_meta{
         collection_id, partition_id, segment_id, field_id, field_schema};
@@ -226,3 +262,72 @@ TEST(JsonContainsByStatsTest, BasicContainsAnyOnArray) {
         EXPECT_EQ(bool(result[i]), should_match);
     }
 }
+
+TEST(JsonStatsUnaryRangeTest, NotEqualKeepsJsonPathErrorsButMasksFieldNull) {
+    auto schema = std::make_shared<Schema>();
+    auto json_fid = schema->AddDebugField("json", DataType::JSON, true);
+
+    auto segment = segcore::CreateSealedSegment(schema);
+
+    std::vector<std::string> json_raw_data = {
+        R"({"a": "1"})",    // equal, filtered out
+        R"({"a": "123"})",  // string mismatch, kept
+        R"({"a": 1})",      // type mismatch for string compare, kept
+        R"({"b": 1})",      // path missing, kept
+        R"({"a": null})",   // JSON path error, kept
+        R"({})",            // path missing, kept
+        R"({"a": "321"})",  // string mismatch, kept
+        R"({"a": "123"})",  // field-level null, filtered out by valid data
+    };
+    std::vector<uint8_t> valid_data{0b01111111};
+
+    const int64_t collection_id = 1101;
+    const int64_t partition_id = 2101;
+    const int64_t segment_id = 3101;
+    const int64_t field_id = json_fid.get();
+    const int64_t build_id = 5101;
+    const int64_t version_id = 1;
+    const std::string root_path = TestLocalPath;
+
+    auto stats = BuildAndLoadJsonKeyStats(json_raw_data,
+                                          json_fid,
+                                          root_path,
+                                          collection_id,
+                                          partition_id,
+                                          segment_id,
+                                          field_id,
+                                          build_id,
+                                          version_id,
+                                          &valid_data);
+    segment->LoadJsonStats(json_fid, stats);
+
+    auto json_field =
+        std::make_shared<FieldData<milvus::Json>>(DataType::JSON, true);
+    json_field->FillFieldData(MakeNullableJsonArray(json_raw_data, valid_data));
+
+    auto cm = milvus::storage::RemoteChunkManagerSingleton::GetInstance()
+                  .GetRemoteChunkManager();
+    auto load_info = PrepareSingleFieldInsertBinlog(
+        0, 0, 0, json_fid.get(), {json_field}, cm);
+    segment->LoadFieldData(load_info);
+
+    proto::plan::GenericValue val;
+    val.set_string_val("1");
+    auto unary_expr = std::make_shared<expr::UnaryRangeFilterExpr>(
+        expr::ColumnInfo(json_fid, DataType::JSON, {"a"}),
+        proto::plan::OpType::NotEqual,
+        val,
+        std::vector<proto::plan::GenericValue>());
+    auto plan =
+        std::make_shared<plan::FilterBitsNode>(DEFAULT_PLANNODE_ID, unary_expr);
+    auto result = query::ExecuteQueryExpr(
+        plan, segment.get(), json_raw_data.size(), MAX_TIMESTAMP);
+
+    ASSERT_EQ(result.size(), json_raw_data.size());
+    EXPECT_FALSE(result[0]);
+    for (int i = 1; i <= 6; ++i) {
+        EXPECT_TRUE(result[i]) << "row " << i;
+    }
+    EXPECT_FALSE(result[7]);
+    EXPECT_EQ(result.count(), 6);
+}

internal/core/src/exec/expression/UnaryExpr.cpp

@@ -871,9 +871,7 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJson(EvalCtx& context) {
                         offset = (offsets) ? offsets[i] : i;
                     }
                     if (valid_data != nullptr && !valid_data[offset]) {
-                        // NULL values are not equal to anything
-                        valid_res[i] = false;
-                        res[i] = true;
+                        res[i] = valid_res[i] = false;
                         continue;
                     }
                     if (has_bitmap_input &&
@@ -1016,33 +1014,54 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
         pinned_json_stats_ = segment->GetJsonStats(op_ctx_, field_id);
         auto* index = pinned_json_stats_.get();
         Assert(index != nullptr);
-        cached_index_chunk_res_ =
-            (op_type == proto::plan::OpType::NotEqual)
-                ? std::make_shared<TargetBitmap>(active_count_, true)
-                : std::make_shared<TargetBitmap>(active_count_);
+        cached_index_chunk_res_ = std::make_shared<TargetBitmap>(active_count_);
         cached_index_chunk_valid_res_ =
             std::make_shared<TargetBitmap>(active_count_, true);
         TargetBitmapView res_view(*cached_index_chunk_res_);
         TargetBitmapView valid_res_view(*cached_index_chunk_valid_res_);
+        int64_t valid_processed_size = 0;
+        for (size_t i = 0;
+             i < num_data_chunk_ && valid_processed_size < active_count_;
+             ++i) {
+            int64_t size = segment_->is_chunked()
+                               ? segment_->chunk_size(field_id_, i)
+                               : (i == num_data_chunk_ - 1
+                                      ? active_count_ - valid_processed_size
+                                      : size_per_chunk_);
+            size =
+                std::min<int64_t>(size, active_count_ - valid_processed_size);
+            if (size <= 0) {
+                continue;
+            }
+            segment_->ApplyFieldValidData(
+                op_ctx_,
+                field_id_,
+                i,
+                0,
+                size,
+                valid_res_view + valid_processed_size);
+            valid_processed_size += size;
+        }
 
         // process shredding data
         auto try_execute = [&](milvus::index::JSONType json_type,
-                               TargetBitmapView& res_view,
-                               TargetBitmapView& valid_res_view,
                                auto GetType,
                                auto ValType) {
             auto target_field = index->GetShreddingField(pointer, json_type);
             if (!target_field.empty()) {
                 using ColType = decltype(GetType);
                 using ValType = decltype(ValType);
+                TargetBitmap target_res(active_count_, false);
+                TargetBitmapView target_res_view(target_res);
                 ShreddingExecutor<ColType, ValType> executor(
                     op_type, pointer, val);
                 index->ExecutorForShreddingData<ColType>(op_ctx_,
                                                          target_field,
                                                          executor,
                                                          nullptr,
-                                                         res_view,
-                                                         valid_res_view);
+                                                         target_res_view,
+                                                         target_res_view);
+                res_view.inplace_or_with_count(target_res_view, active_count_);
                 LOG_DEBUG(
                     "using shredding data's field: {} with value {}, count {} "
                     "for segment {}",
@@ -1059,71 +1078,42 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
                 [this](double us) { json_stats_shredding_latency_us_ += us; });
 
             if constexpr (std::is_same_v<GetType, bool>) {
-                try_execute(milvus::index::JSONType::BOOL,
-                            res_view,
-                            valid_res_view,
-                            bool{},
-                            bool{});
+                try_execute(milvus::index::JSONType::BOOL, bool{}, bool{});
             } else if constexpr (std::is_same_v<GetType, int64_t>) {
-                try_execute(milvus::index::JSONType::INT64,
-                            res_view,
-                            valid_res_view,
-                            int64_t{},
-                            int64_t{});
+                try_execute(
+                    milvus::index::JSONType::INT64, int64_t{}, int64_t{});
 
                 // and double compare
-                TargetBitmap res_double(active_count_, false);
-                TargetBitmapView res_double_view(res_double);
-                TargetBitmap res_double_valid(active_count_, true);
-                TargetBitmapView valid_res_double_view(res_double_valid);
-                try_execute(milvus::index::JSONType::DOUBLE,
-                            res_double_view,
-                            valid_res_double_view,
-                            double{},
-                            int64_t{});
-                res_view.inplace_or_with_count(res_double_view, active_count_);
-                valid_res_view.inplace_or_with_count(valid_res_double_view,
-                                                     active_count_);
+                try_execute(
+                    milvus::index::JSONType::DOUBLE, double{}, int64_t{});
             } else if constexpr (std::is_same_v<GetType, double>) {
-                try_execute(milvus::index::JSONType::DOUBLE,
-                            res_view,
-                            valid_res_view,
-                            double{},
-                            double{});
+                try_execute(
+                    milvus::index::JSONType::DOUBLE, double{}, double{});
 
                 // add int64 compare
-                TargetBitmap res_int64(active_count_, false);
-                TargetBitmapView res_int64_view(res_int64);
-                TargetBitmap res_int64_valid(active_count_, true);
-                TargetBitmapView valid_res_int64_view(res_int64_valid);
-                try_execute(milvus::index::JSONType::INT64,
-                            res_int64_view,
-                            valid_res_int64_view,
-                            int64_t{},
-                            double{});
-                res_view.inplace_or_with_count(res_int64_view, active_count_);
-                valid_res_view.inplace_or_with_count(valid_res_int64_view,
-                                                     active_count_);
+                try_execute(
+                    milvus::index::JSONType::INT64, int64_t{}, double{});
             } else if constexpr (std::is_same_v<GetType, std::string> ||
                                  std::is_same_v<GetType, std::string_view>) {
-                try_execute(milvus::index::JSONType::STRING,
-                            res_view,
-                            valid_res_view,
-                            GetType{},
-                            GetType{});
+                try_execute(
+                    milvus::index::JSONType::STRING, GetType{}, GetType{});
             } else if constexpr (std::is_same_v<GetType, proto::plan::Array>) {
                 // ARRAY shredding data: stored as BSON binary in binary column
                 auto target_field = index->GetShreddingField(
                     pointer, milvus::index::JSONType::ARRAY);
                 if (!target_field.empty()) {
+                    TargetBitmap target_res(active_count_, false);
+                    TargetBitmapView target_res_view(target_res);
                     ShreddingArrayBsonExecutor executor(op_type, pointer, val);
                     index->template ExecutorForShreddingData<std::string_view>(
                         op_ctx_,
                         target_field,
                         executor,
                         nullptr,
-                        res_view,
-                        valid_res_view);
+                        target_res_view,
+                        target_res_view);
+                    res_view.inplace_or_with_count(target_res_view,
+                                                   active_count_);
                     LOG_DEBUG("using shredding array field: {}, count {}",
                               target_field,
                               res_view.count());
@@ -1136,50 +1126,51 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
                                    milvus::BsonView bson,
                                    uint32_t row_id,
                                    uint32_t value_offset) {
+            auto set_unknown = [&](uint32_t row_id) {
+                res_view[row_id] = false;
+            };
+            auto set_known = [&](uint32_t row_id, bool value) {
+                res_view[row_id] = value;
+            };
             if constexpr (std::is_same_v<GetType, proto::plan::Array>) {
                 Assert(op_type == proto::plan::OpType::Equal ||
                        op_type == proto::plan::OpType::NotEqual);
                 if (array_index != INVALID_ARRAY_INDEX) {
                     auto array_value = bson.ParseAsArrayAtOffset(value_offset);
                     if (!array_value.has_value()) {
-                        // For NotEqual: path not exists means "not equal", keep true
-                        // For Equal: path not exists means no match, set false
-                        res_view[row_id] =
-                            (op_type == proto::plan::OpType::NotEqual);
+                        set_unknown(row_id);
                         return;
                     }
                     auto sub_array = milvus::BsonView::GetNthElementInArray<
                         bsoncxx::array::view>(array_value.value().data(),
                                               array_index);
                     if (!sub_array.has_value()) {
-                        res_view[row_id] =
-                            (op_type == proto::plan::OpType::NotEqual);
+                        set_unknown(row_id);
                         return;
                     }
-                    res_view[row_id] =
+                    set_known(
+                        row_id,
                         op_type == proto::plan::OpType::Equal
                             ? CompareTwoJsonArray(sub_array.value(), val)
-                            : !CompareTwoJsonArray(sub_array.value(), val);
+                            : !CompareTwoJsonArray(sub_array.value(), val));
                 } else {
                     auto array_value = bson.ParseAsArrayAtOffset(value_offset);
                     if (!array_value.has_value()) {
-                        res_view[row_id] =
-                            (op_type == proto::plan::OpType::NotEqual);
+                        set_unknown(row_id);
                         return;
                     }
-                    res_view[row_id] =
+                    set_known(
+                        row_id,
                         op_type == proto::plan::OpType::Equal
                             ? CompareTwoJsonArray(array_value.value(), val)
-                            : !CompareTwoJsonArray(array_value.value(), val);
+                            : !CompareTwoJsonArray(array_value.value(), val));
                 }
             } else {
                 std::optional<GetType> get_value;
                 if (array_index != INVALID_ARRAY_INDEX) {
                     auto array_value = bson.ParseAsArrayAtOffset(value_offset);
                     if (!array_value.has_value()) {
-                        // Path not exists: NotEqual->true, others->false
-                        res_view[row_id] =
-                            (op_type == proto::plan::OpType::NotEqual);
+                        set_unknown(row_id);
                         return;
                     }
                     get_value = milvus::BsonView::GetNthElementInArray<GetType>(
@@ -1191,12 +1182,11 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
                                 milvus::BsonView::GetNthElementInArray<double>(
                                     array_value.value().data(), array_index);
                             if (get_value.has_value()) {
-                                res_view[row_id] = UnaryCompare(
-                                    get_value.value(), val, op_type);
+                                set_known(row_id,
+                                          UnaryCompare(
+                                              get_value.value(), val, op_type));
                             } else {
-                                // Type mismatch: NotEqual->true, others->false
-                                res_view[row_id] =
-                                    (op_type == proto::plan::OpType::NotEqual);
+                                set_unknown(row_id);
                             }
                             return;
                         }
@@ -1206,11 +1196,11 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
                                 milvus::BsonView::GetNthElementInArray<int64_t>(
                                     array_value.value().data(), array_index);
                             if (get_value.has_value()) {
-                                res_view[row_id] = UnaryCompare(
-                                    get_value.value(), val, op_type);
+                                set_known(row_id,
+                                          UnaryCompare(
+                                              get_value.value(), val, op_type));
                             } else {
-                                res_view[row_id] =
-                                    (op_type == proto::plan::OpType::NotEqual);
+                                set_unknown(row_id);
                             }
                             return;
                         }
@@ -1224,11 +1214,11 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
                             auto get_value =
                                 bson.ParseAsValueAtOffset<double>(value_offset);
                             if (get_value.has_value()) {
-                                res_view[row_id] = UnaryCompare(
-                                    get_value.value(), val, op_type);
+                                set_known(row_id,
+                                          UnaryCompare(
+                                              get_value.value(), val, op_type));
                             } else {
-                                res_view[row_id] =
-                                    (op_type == proto::plan::OpType::NotEqual);
+                                set_unknown(row_id);
                             }
                             return;
                         }
@@ -1237,23 +1227,22 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
                             auto get_value = bson.ParseAsValueAtOffset<int64_t>(
                                 value_offset);
                             if (get_value.has_value()) {
-                                res_view[row_id] = UnaryCompare(
-                                    get_value.value(), val, op_type);
+                                set_known(row_id,
+                                          UnaryCompare(
+                                              get_value.value(), val, op_type));
                             } else {
-                                res_view[row_id] =
-                                    (op_type == proto::plan::OpType::NotEqual);
+                                set_unknown(row_id);
                             }
                             return;
                         }
                     }
                 }
                 if (!get_value.has_value()) {
-                    res_view[row_id] =
-                        (op_type == proto::plan::OpType::NotEqual);
+                    set_unknown(row_id);
                     return;
                 }
-                res_view[row_id] =
-                    UnaryCompare(get_value.value(), val, op_type);
+                set_known(row_id,
+                          UnaryCompare(get_value.value(), val, op_type));
             }
         };
 
@@ -1280,6 +1269,7 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() {
         if (expr_->op_type_ == proto::plan::OpType::NotEqual) {
             cached_index_chunk_res_->flip();
         }
+        res_view.inplace_and(valid_res_view, active_count_);
         cached_index_chunk_id_ = 0;
     }
 

internal/core/src/exec/operator/FilterBitsNode.cpp

@@ -23,6 +23,25 @@
 
 namespace milvus {
 namespace exec {
+
+namespace {
+
+void
+ConvertPredicateToFilteredBitset(TargetBitmapView data,
+                                 TargetBitmapView valid,
+                                 const size_t size) {
+    // FilterBitsNode outputs a filtered-row bitset: 1 means excluded. A SQL-style
+    // predicate passes only when it is definitely TRUE, so UNKNOWN/NULL must be
+    // excluded together with FALSE.
+    data.flip();
+    TargetBitmap invalid(valid);
+    invalid.flip();
+    data.inplace_or(invalid, size);
+    valid.set();
+}
+
+}  // namespace
+
 PhyFilterBitsNode::PhyFilterBitsNode(
     int32_t operator_id,
     DriverContext* driverctx,
@@ -119,7 +138,11 @@ PhyFilterBitsNode::GetOutput() {
                       "PhyFilterBitsNode result should be ColumnVector");
         }
     }
-    bitset.flip();
+    TargetBitmapView bitset_view(bitset);
+    TargetBitmapView valid_bitset_view(valid_bitset);
+    ConvertPredicateToFilteredBitset(
+        bitset_view, valid_bitset_view, bitset.size());
+
     AssertInfo(bitset.size() == need_process_rows_,
                "bitset size: {}, need_process_rows_: {}",
                bitset.size(),

internal/core/src/index/TextMatchIndexTest.cpp

@@ -1201,6 +1201,7 @@ TEST(TextMatch, ConcurrentReadWriteWithNull) {
     for (int64_t i = 0; i < N - 1; i++) {
         str_col_valid->at(i) = false;
     }
+    str_col_valid->at(N - 1) = true;
 
     std::thread writer([&seg, &raw_data, N]() {
         seg->PreInsert(N);
@@ -1216,7 +1217,7 @@ TEST(TextMatch, ConcurrentReadWriteWithNull) {
         ;
         const std::chrono::seconds timeout_duration{2};
         while (true) {
-            if (start - std::chrono::high_resolution_clock::now() >
+            if (std::chrono::high_resolution_clock::now() - start >
                 timeout_duration) {
                 ASSERT_TRUE(false)
                     << "Failed to get valid results within timeout";