From ef166a673a7f584ddda0215360a95e9ae92c73fd Mon Sep 17 00:00:00 2001 From: James <83447078+xiaofan-luan@users.noreply.github.com> Date: Fri, 19 Jun 2026 20:27:10 -0700 Subject: [PATCH] enhance: support building core on macOS (Apple Silicon) with clang 19 (#50616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit issue: #50615 ## What this PR does Makes the C++ core build cleanly on **macOS (Apple Silicon) with Homebrew LLVM/clang 19**. > [!IMPORTANT] > **Scope / impact is NOT macOS-only.** Although this PR is framed as macOS/clang-19 enablement, two of the changes are **top-level `requires(...)` in `internal/core/conanfile.py`, so they apply to every platform, including Linux production builds:** > - **`azure-sdk-for-cpp` `1.11.3@milvus/dev` → `1.16.0@milvus/dev`** (5 minor versions, `force=True`). This is the SDK Arrow uses for the **Azure Blob remote-storage path**, so this is effectively a major Azure SDK bump on the **production Azure object-storage path for Linux deployments** too. > - **`arrow` recipe `17.0.0@milvus/dev-2.6#c743ea7a…` → `17.0.0@milvus/dev#f411aa73…`** — same Arrow version, but **rebuilt against azure 1.16.0** with a patched `FindAzure.cmake`. The Arrow **`package_id` (recipe revision) DID change**; it is not the same binary as before. > - **`libbson/1.30.6@milvus/dev`** added (also global, but new — no pre-existing behavior to regress). > > **Testing blind spot:** CI green proves compile/link/UT/e2e pass, but the UT and integration suites **do not exercise real Azure Blob read/write**, so the Azure object-storage path is **not covered** by this PR's CI. A real Azure Blob remote-storage regression is recommended before merge (see Verification). ### Why the dependency bumps are required for clang 19 - **azure 1.16.0**: azure 1.11.3 bundles an older `nlohmann/json` that uses `std::char_traits`, which was **removed in libc++ 19** → 1.11.3 fails to compile on clang 19. 1.16.0 fixes this. - **arrow rebuilt against azure 1.16.0**: when Arrow is built from source on clang 19 it must link the same azure 1.16.0; the recipe also patches `FindAzure.cmake` to the monolithic `Azure` config so `find_package(Azure)` resolves. Hence a new Arrow recipe revision. ### Toolchain / build enablement - **`scripts/setenv.sh`**: probe LLVM 19/20/21 first (libc++ 17/18 `chrono operator<<` clashes with Arrow's vendored `date`; libc++ 19 fixes it). - **`scripts/3rdparty_build.sh`**: arm64 `-march=armv8-a+crypto+crc` so folly's F14 (SimdAndCrc on Apple Silicon) matches core/knowhere — fixes `F14LinkCheck` undefined-symbol link errors. - **`InvertedIndexTantivy.cpp`**: drop spurious `->template` on non-template members (clang 19 `-Wmissing-template-arg-list-after-template-kw`). ### Drop bsoncxx / libmongoc → libbson only The BSON layer used the **bsoncxx** C++ driver, whose CMake unconditionally builds mongo-cxx-driver → **libmongoc** → bundled **utf8proc**, which fails to configure under newer CMake/clang on macOS. Milvus only needs BSON (de)serialization. - New Conan package **`libbson/1.30.6@milvus/dev`** (libbson-only, `ENABLE_MONGOC=OFF`; recipe published to milvus conanfiles & production). - New **`src/common/bson_shim.h`** — a thin `milvus::bson` C++ layer over libbson's C API. - Migrated `bson_view.h`, `bson_builder.{h,cpp}`, json_stats consumers and unit tests off bsoncxx. - CMake: `find_package(bson-1.0)` + link `mongo::bson_shared`; removed FetchContent `thirdparty/bsoncxx`. All three global Conan requires (arrow / libbson / azure) pin an explicit recipe `#revision` for reproducibility. ## Verification - Full clean rebuild on macOS arm64 / clang 19: `bin/milvus` builds; `libmilvus_core` links **only `libbson-1.0.dylib`** (no libmongoc/mongocxx/utf8proc). - libbson / arrow / azure resolve from production Conan at the pinned revisions. - All 21 BSON unit tests (BsonView / BsonBuilder / DomNode) pass. - Full ci-v2 suite green on Linux (build / build-ut-cov / ut-cpp / ut-go / integration / e2e / go-sdk). - **TODO before merge:** real Azure Blob remote-storage read/write regression to cover the azure-1.16.0 / arrow-rebuild blast radius (not exercised by UT/integration). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: xiaofanluan Co-authored-by: xiaofanluan Co-authored-by: Claude Opus 4.8 (1M context) --- internal/core/CMakeLists.txt | 8 +- internal/core/conanfile.py | 7 +- internal/core/lsan_suppressions.txt | 8 + internal/core/src/CMakeLists.txt | 3 +- internal/core/src/common/bson_shim.h | 336 +++++++++++++++ internal/core/src/common/bson_view.h | 183 ++++---- .../src/exec/expression/JsonContainsExpr.cpp | 10 +- .../src/exec/expression/JsonContainsExpr.h | 10 +- .../core/src/exec/expression/UnaryExpr.cpp | 6 +- internal/core/src/exec/expression/Utils.h | 18 +- .../core/src/index/InvertedIndexTantivy.cpp | 23 +- .../src/index/json_stats/JsonKeyStats.cpp | 19 +- .../src/index/json_stats/bson_builder.cpp | 275 ++++++------ .../core/src/index/json_stats/bson_builder.h | 67 ++- internal/core/thirdparty/CMakeLists.txt | 1 - .../core/thirdparty/bsoncxx/CMakeLists.txt | 63 --- .../test_json_stats/test_bson_builder.cpp | 400 ++++++++++-------- .../test_json_stats/test_bson_view.cpp | 323 ++++++++------ scripts/3rdparty_build.sh | 5 +- scripts/setenv.sh | 6 +- 20 files changed, 1100 insertions(+), 671 deletions(-) create mode 100644 internal/core/src/common/bson_shim.h delete mode 100644 internal/core/thirdparty/bsoncxx/CMakeLists.txt diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index 3e34ecb2a1..cb4c3ff376 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -143,10 +143,9 @@ set( MILVUS_THIRDPARTY_SRC ${PROJECT_SOURCE_DIR}/thirdparty ) # self-installed dynamic libraries will be correctly linked by excutable set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" "${CMAKE_INSTALL_PREFIX}/lib64" ) set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE ) -# FetchContent-built shared libs (libbson, libbsoncxx, etc.) are placed in -# ${CMAKE_BINARY_DIR}/lib. The linker needs -rpath-link to resolve transitive -# shared library dependencies (e.g. libbsoncxx.so -> libbson-1.0.so) when -# linking test binaries. +# The FetchContent-built libbson shared lib is placed in ${CMAKE_BINARY_DIR}/lib. +# The linker needs -rpath-link to resolve it (e.g. libmilvus_core.so -> +# libbson-1.0.so) when linking test binaries. set( CMAKE_BUILD_RPATH "${CMAKE_BINARY_DIR}/lib" ) # **************************** Dependencies **************************** @@ -186,6 +185,7 @@ find_package(gflags REQUIRED) find_package(xsimd REQUIRED) find_package(unordered_dense REQUIRED) find_package(absl REQUIRED) +find_package(bson-1.0 REQUIRED) find_package(simde REQUIRED) find_package(milvus-common REQUIRED) if (LINUX) diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py index f2896ca350..a8ed220b9f 100644 --- a/internal/core/conanfile.py +++ b/internal/core/conanfile.py @@ -12,7 +12,7 @@ class MilvusConan(ConanFile): "rocksdb/6.29.5@milvus/dev#67b8ae76ad7be5f779082f67416f89bf", "onetbb/2021.9.0#f9d7a3aa294ac4a594a93f9b4c7f272d", "zstd/1.5.5#70dc5eb8ea16708fc946fbac884c507e", - "arrow/17.0.0@milvus/dev-2.6#c743ea7a6f2420ba5811b2be3df59892", + "arrow/17.0.0@milvus/dev#f411aa733829a554644f281ccb2ae0a8", "libevent/2.1.12#95065aaefcd58d3956d6dfbfc5631d97", "googleapis/cci.20221108#4553d68a2429cc0fff7d2bab4e5b3ea9", "gtest/1.13.0#2cf98fac7337eb73fc4ee839dbcd4468", @@ -123,10 +123,13 @@ class MilvusConan(ConanFile): self.requires("nlohmann_json/3.11.3#ffb9e9236619f1c883e36662f944345d", force=True) self.requires("abseil/20250127.0#481edcc75deb0efb16500f511f0f0a1c", force=True) self.requires("fmt/11.2.0#eb98daa559c7c59d591f4720dde4cd5c", force=True) + # libbson only (BSON C library) for JSON stats — NOT the full mongo-c-driver. + # Drops libmongoc/mongocxx/utf8proc; see src/common/bson_shim.h. + self.requires("libbson/1.30.6@milvus/dev#4fc4c269cbda1b46c3118fa396cdc690") # azure-sdk-for-cpp is a transitive dep of Arrow, but must be declared # as a direct dep so CMakeDeps generates standalone cmake config files. # Without this, find_package(Azure) can't find include directories. - self.requires("azure-sdk-for-cpp/1.11.3@milvus/dev#395e8e7a0c29644d41ef160088128f14") + self.requires("azure-sdk-for-cpp/1.16.0@milvus/dev#9e2475502f8ee3b284c9e0731a3370c6", force=True) self.requires("aws-sdk-cpp/1.11.692@milvus/dev#c309ce91fa572fff68f9f4e36d477a04") # Force snappy/lz4 versions to override Arrow's older transitive deps # (arrow/*:with_snappy and arrow/*:with_lz4 are enabled for Parquet decoding) diff --git a/internal/core/lsan_suppressions.txt b/internal/core/lsan_suppressions.txt index 7213d32b43..4df92d7e32 100644 --- a/internal/core/lsan_suppressions.txt +++ b/internal/core/lsan_suppressions.txt @@ -20,3 +20,11 @@ leak:__cxa_allocate_exception # https://github.com/grpc/grpc/issues/24488 leak:_GLOBAL__sub_I_audit_logging.cc leak:grpc_core::experimental::AuditLoggerRegistry + +# libcurl "share" handle: milvus-storage's S3 client initializes a global CURLSH +# (shared connection/DNS cache) once and intentionally never calls +# curl_share_cleanup, so it lives for the whole process. In the uninstrumented +# libmilvus-storage.so this reachable global is reported as a leak at exit. +# Not a bug. +leak:curl_share_init +leak:curl_share_setopt diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index 342ba1f1f3..9208be5b41 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -32,7 +32,6 @@ include_directories( ${KNOWHERE_INCLUDE_DIR} ${SIMDJSON_INCLUDE_DIR} ${TANTIVY_INCLUDE_DIR} - ${BSONCXX_INCLUDE_DIR} ${MILVUS_STORAGE_INCLUDE_DIR} ) @@ -71,6 +70,7 @@ set(CONAN_TARGETS xsimd unordered_dense::unordered_dense abseil::abseil + mongo::bson_shared simde::simde milvus-common::milvus-common LibLZMA::LibLZMA @@ -163,7 +163,6 @@ set(LINK_TARGETS tantivy_binding knowhere milvus-storage - bsoncxx ${OpenMP_CXX_FLAGS} ${CONAN_TARGETS} ) diff --git a/internal/core/src/common/bson_shim.h b/internal/core/src/common/bson_shim.h new file mode 100644 index 0000000000..148335e269 --- /dev/null +++ b/internal/core/src/common/bson_shim.h @@ -0,0 +1,336 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// A minimal C++ layer over libbson's C API, covering exactly the BSON subset +// Milvus uses for JSON (de)serialization. It replaces the former dependency on +// the bsoncxx C++ driver, which transitively pulled in libmongoc + utf8proc. +// +// The view types here are NON-OWNING: they reference bytes in a BSON buffer the +// caller must keep alive (same semantics as the bsoncxx ::view types they +// replace). libbson's bson_iter_t reads through offsets into the external +// buffer, so copied iterators / copied bson_value_t views remain valid as long +// as that buffer outlives them. + +#include + +#include +#include +#include +#include + +#include "fmt/format.h" + +namespace milvus::bson { + +// BSON element type tags. Numeric values match the BSON spec exactly (identical +// to libbson's bson_type_t and the former bsoncxx::type), so they can be +// compared directly against the raw type byte in a BSON buffer. +enum class type : uint8_t { + k_double = 0x01, + k_string = 0x02, // BSON_TYPE_UTF8 + k_document = 0x03, + k_array = 0x04, + k_binary = 0x05, + k_bool = 0x08, + k_null = 0x0A, + k_int32 = 0x10, + k_int64 = 0x12, +}; + +// Scalar accessor results mirror bsoncxx's b_*{ value } shape so call sites can +// keep using `.value`. +struct b_int32 { + int32_t value; +}; +struct b_int64 { + int64_t value; +}; +struct b_double { + double value; +}; +struct b_bool { + bool value; +}; +struct b_string { + std::string_view value; +}; + +class array_view; +class document_view; +class value_view; + +namespace detail { +// Shared scalar accessors over a copied bson_value_t (used by both value_view +// and element). +struct value_accessors { + bson_value_t v_{}; + + bson::type + type() const { + return static_cast(v_.value_type); + } + b_int32 + get_int32() const { + return {v_.value.v_int32}; + } + b_int64 + get_int64() const { + return {v_.value.v_int64}; + } + b_double + get_double() const { + return {v_.value.v_double}; + } + b_bool + get_bool() const { + return {v_.value.v_bool}; + } + b_string + get_string() const { + return {std::string_view(v_.value.v_utf8.str, v_.value.v_utf8.len)}; + } +}; +} // namespace detail + +// A non-owning view over one key/value pair produced during iteration. Mirrors +// the subset of bsoncxx::document::element / array element Milvus relies on. +// get_value() is defined out-of-line below, once value_view is complete. +class element : public detail::value_accessors { + public: + element() = default; + element(std::string_view key, const bson_value_t& v) : key_(key) { + v_ = v; + } + + // bsoncxx key() returns a string_view-like object exposing data()/length(). + std::string_view + key() const { + return key_; + } + value_view + get_value() const; + + private: + std::string_view key_; +}; + +// A non-owning view over a BSON document or array buffer (length-prefixed BSON +// bytes). array_view and document_view are distinct types (so overloads / +// template specializations can tell them apart) but iterate identically — a +// BSON array is just a document with numeric keys. +class view_base { + public: + view_base() = default; + view_base(const uint8_t* data, size_t len) + : data_(data), len_(static_cast(len)) { + } + + const uint8_t* + data() const { + return data_; + } + uint32_t + length() const { + return len_; + } + + class iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = element; + using difference_type = std::ptrdiff_t; + using pointer = const element*; + using reference = const element&; + + iterator() = default; // end sentinel + + iterator(const uint8_t* data, uint32_t len) { + if (data == nullptr || len == 0 || + !bson_init_static(&bson_, data, len) || + !bson_iter_init(&iter_, &bson_)) { + valid_ = false; + return; + } + advance(); + } + + const element& + operator*() const { + return cur_; + } + const element* + operator->() const { + return &cur_; + } + iterator& + operator++() { + advance(); + return *this; + } + bool + operator==(const iterator& o) const { + if (valid_ != o.valid_) { + return false; + } + if (!valid_) { + return true; + } + return iter_.off == o.iter_.off; + } + bool + operator!=(const iterator& o) const { + return !(*this == o); + } + + private: + void + advance() { + if (bson_iter_next(&iter_)) { + cur_ = element(std::string_view(bson_iter_key(&iter_)), + *bson_iter_value(&iter_)); + valid_ = true; + } else { + valid_ = false; + } + } + + bson_t bson_{}; + bson_iter_t iter_{}; + element cur_{}; + bool valid_{false}; + }; + + iterator + begin() const { + return iterator(data_, len_); + } + iterator + end() const { + return iterator(); + } + + protected: + const uint8_t* data_{nullptr}; + uint32_t len_{0}; +}; + +class document_view : public view_base { + public: + using view_base::view_base; +}; + +class array_view : public view_base { + public: + using view_base::view_base; +}; + +// A non-owning view over a single BSON value. Defined after the view types so +// its document_holder / array_holder can hold complete document_view / +// array_view members. +class value_view : public detail::value_accessors { + public: + value_view() { + v_.value_type = BSON_TYPE_EOD; + } + explicit value_view(const bson_value_t& v) { + v_ = v; + } + + struct document_holder { + document_view value; + document_view + view() const { + return value; + } + }; + struct array_holder { + array_view value; + }; + + document_holder + get_document() const { + return {document_view(v_.value.v_doc.data, v_.value.v_doc.data_len)}; + } + array_holder + get_array() const { + return {array_view(v_.value.v_doc.data, v_.value.v_doc.data_len)}; + } +}; + +// element::get_value() needs value_view complete. +inline value_view +element::get_value() const { + return value_view(v_); +} + +// Serialize a BSON document to relaxed extended JSON (debug / logging use). +inline std::string +to_json(const document_view& doc) { + bson_t b; + if (doc.data() == nullptr || + !bson_init_static(&b, doc.data(), doc.length())) { + return "{}"; + } + size_t len = 0; + char* s = bson_as_relaxed_extended_json(&b, &len); + if (s == nullptr) { + return "{}"; + } + std::string out(s, len); + bson_free(s); + return out; +} + +} // namespace milvus::bson + +template <> +struct fmt::formatter : fmt::formatter { + auto + format(milvus::bson::type t, fmt::format_context& ctx) const { + std::string name; + switch (t) { + case milvus::bson::type::k_int32: + name = "int32"; + break; + case milvus::bson::type::k_int64: + name = "int64"; + break; + case milvus::bson::type::k_double: + name = "double"; + break; + case milvus::bson::type::k_string: + name = "string"; + break; + case milvus::bson::type::k_bool: + name = "bool"; + break; + case milvus::bson::type::k_null: + name = "null"; + break; + case milvus::bson::type::k_document: + name = "document"; + break; + case milvus::bson::type::k_array: + name = "array"; + break; + default: + name = "Unknown"; + } + return fmt::formatter::format(name, ctx); + } +}; diff --git a/internal/core/src/common/bson_view.h b/internal/core/src/common/bson_view.h index c710c8d0b6..958842ba01 100644 --- a/internal/core/src/common/bson_view.h +++ b/internal/core/src/common/bson_view.h @@ -16,12 +16,6 @@ #pragma once -#include -#include -#include -#include -#include -#include #include #include #include @@ -36,50 +30,14 @@ #include #include "common/EasyAssert.h" +#include "common/bson_shim.h" #include "fmt/format.h" #include "log/Log.h" -template <> -struct fmt::formatter : fmt::formatter { - auto - format(bsoncxx::type type, fmt::format_context& ctx) const { - std::string name; - switch (type) { - case bsoncxx::type::k_int32: - name = "int32"; - break; - case bsoncxx::type::k_int64: - name = "int64"; - break; - case bsoncxx::type::k_double: - name = "double"; - break; - case bsoncxx::type::k_string: - name = "string"; - break; - case bsoncxx::type::k_bool: - name = "bool"; - break; - case bsoncxx::type::k_null: - name = "null"; - break; - case bsoncxx::type::k_document: - name = "document"; - break; - case bsoncxx::type::k_array: - name = "array"; - break; - default: - name = "Unknown"; - } - return fmt::formatter::format(name, ctx); - } -}; - namespace milvus { struct BsonRawField { - bsoncxx::type type; + milvus::bson::type type; std::string key; const uint8_t* value_ptr; // points to value (not including type/key) }; @@ -124,16 +82,16 @@ ReadRawDocOrArray(const uint8_t* ptr) { return std::vector(ptr, ptr + len); } -inline bsoncxx::document::view +inline milvus::bson::document_view ParseAsDocument(const uint8_t* ptr) { int32_t len = *reinterpret_cast(ptr); - return bsoncxx::document::view(ptr, len); + return milvus::bson::document_view(ptr, len); } -inline bsoncxx::array::view +inline milvus::bson::array_view ParseAsArray(const uint8_t* ptr) { int32_t len = *reinterpret_cast(ptr); - return bsoncxx::array::view(ptr, len); + return milvus::bson::array_view(ptr, len); } template @@ -211,13 +169,13 @@ class BsonView { // Core implementation: check if a BSON value is "empty" (null or recursively contains only nulls/empties) // Following the same semantics as Json::isObjectEmpty/isDocEmpty in Json.h static bool - IsBsonValueEmpty(const bsoncxx::types::bson_value::view& val) { + IsBsonValueEmpty(const milvus::bson::value_view& val) { switch (val.type()) { - case bsoncxx::type::k_null: + case milvus::bson::type::k_null: return true; - case bsoncxx::type::k_document: + case milvus::bson::type::k_document: return IsBsonValueEmpty(val.get_document().value); - case bsoncxx::type::k_array: + case milvus::bson::type::k_array: return IsBsonValueEmpty(val.get_array().value); default: return false; @@ -225,7 +183,7 @@ class BsonView { } static bool - IsBsonValueEmpty(const bsoncxx::document::view& doc) { + IsBsonValueEmpty(const milvus::bson::document_view& doc) { for (auto&& elem : doc) { if (!IsBsonValueEmpty(elem.get_value())) { return false; @@ -235,7 +193,7 @@ class BsonView { } static bool - IsBsonValueEmpty(const bsoncxx::array::view& arr) { + IsBsonValueEmpty(const milvus::bson::array_view& arr) { for (auto&& elem : arr) { if (!IsBsonValueEmpty(elem.get_value())) { return false; @@ -250,11 +208,11 @@ class BsonView { auto field = ParseBsonField(data_, offset); switch (field.type) { - case bsoncxx::type::k_null: + case milvus::bson::type::k_null: return true; - case bsoncxx::type::k_document: + case milvus::bson::type::k_document: return IsBsonValueEmpty(ParseAsDocument(field.value_ptr)); - case bsoncxx::type::k_array: + case milvus::bson::type::k_array: return IsBsonValueEmpty(ParseAsArray(field.value_ptr)); default: return false; @@ -270,8 +228,8 @@ class BsonView { std::string ToString() const { - bsoncxx::document::view view(data_, size_); - return bsoncxx::to_json(view); + milvus::bson::document_view view(data_, size_); + return milvus::bson::to_json(view); } bool @@ -279,8 +237,8 @@ class BsonView { const uint8_t* ptr = data_ + offset; AssertInfo(offset < size_, "bson offset out of range"); - auto type_tag = static_cast(*ptr++); - return type_tag == bsoncxx::type::k_null; + auto type_tag = static_cast(*ptr++); + return type_tag == milvus::bson::type::k_null; } template @@ -295,7 +253,7 @@ class BsonView { offset < size_, "bson offset:{} out of range:{}", offset, size_); // parse type tag - auto type_tag = static_cast(*ptr++); + auto type_tag = static_cast(*ptr++); // parse key const char* key_cstr = reinterpret_cast(ptr); @@ -304,7 +262,7 @@ class BsonView { // parse value switch (type_tag) { - case bsoncxx::type::k_int32: + case milvus::bson::type::k_int32: if constexpr (std::is_same_v) { return GetValue(ptr); } @@ -312,14 +270,14 @@ class BsonView { return static_cast(GetValue(ptr)); } break; - case bsoncxx::type::k_int64: + case milvus::bson::type::k_int64: if constexpr (std::is_same_v) { return GetValue(ptr); } else if constexpr (std::is_same_v) { return static_cast(GetValue(ptr)); } break; - case bsoncxx::type::k_double: + case milvus::bson::type::k_double: if constexpr (std::is_same_v) { return GetValue(ptr); } @@ -329,12 +287,12 @@ class BsonView { // } // } break; - case bsoncxx::type::k_bool: + case milvus::bson::type::k_bool: if constexpr (std::is_same_v) { return GetValue(ptr); } break; - case bsoncxx::type::k_string: + case milvus::bson::type::k_string: if (ptr + 4 > data_ + size_) { return std::nullopt; } @@ -345,9 +303,9 @@ class BsonView { return GetValue(ptr); } break; - case bsoncxx::type::k_null: - case bsoncxx::type::k_document: - case bsoncxx::type::k_array: + case milvus::bson::type::k_null: + case milvus::bson::type::k_document: + case milvus::bson::type::k_array: break; default: ThrowInfo( @@ -356,18 +314,19 @@ class BsonView { return std::nullopt; } - std::optional + std::optional ParseAsArrayAtOffset(size_t offset) { if (offset == 0) { // if offset is 0, it means the array is the whole bson_view - return bsoncxx::array::view(data_, size_); + return milvus::bson::array_view(data_, size_); } // check offset AssertInfo(offset < size_, "bson offset out of range"); const uint8_t* ptr = data_ + offset; - if (static_cast(*ptr) != bsoncxx::type::k_array) { + if (static_cast(*ptr) != + milvus::bson::type::k_array) { return std::nullopt; } ptr++; @@ -383,11 +342,11 @@ class BsonView { ThrowInfo(ErrorCode::UnexpectedError, "ParseAsArrayAtOffset out of range"); } - return bsoncxx::array::view(view_start, len); + return milvus::bson::array_view(view_start, len); } - inline std::optional - FindByPath(const bsoncxx::document::view& doc_view, + inline std::optional + FindByPath(const milvus::bson::document_view& doc_view, const std::vector& path, size_t idx = 0) { if (idx >= path.size()) @@ -404,11 +363,11 @@ class BsonView { // Recursively process nested structures switch (value.type()) { - case bsoncxx::type::k_document: { + case milvus::bson::type::k_document: { auto sub_doc = value.get_document(); return FindByPath(sub_doc.view(), path, idx + 1); } - case bsoncxx::type::k_array: + case milvus::bson::type::k_array: // TODO: may support array index from parent json for now break; default: @@ -432,28 +391,28 @@ class BsonView { ++ptr; if (i == index) { - switch (static_cast(type_tag)) { - case bsoncxx::type::k_int32: + switch (static_cast(type_tag)) { + case milvus::bson::type::k_int32: if constexpr (std::is_same_v) { return ReadInt32(ptr); } break; - case bsoncxx::type::k_int64: + case milvus::bson::type::k_int64: if constexpr (std::is_same_v) { return ReadInt64(ptr); } break; - case bsoncxx::type::k_double: + case milvus::bson::type::k_double: if constexpr (std::is_same_v) { return ReadDouble(ptr); } break; - case bsoncxx::type::k_bool: + case milvus::bson::type::k_bool: if constexpr (std::is_same_v) { return ReadBool(ptr); } break; - case bsoncxx::type::k_string: + case milvus::bson::type::k_string: if constexpr (std::is_same_v) { return ReadUtf8(ptr); } @@ -461,8 +420,10 @@ class BsonView { return ReadUtf8View(ptr); } break; - case bsoncxx::type::k_array: - if constexpr (std::is_same_v) { + case milvus::bson::type::k_array: + if constexpr (std::is_same_v< + T, + milvus::bson::array_view>) { return ParseAsArray(ptr); } break; @@ -471,26 +432,26 @@ class BsonView { } } - switch (static_cast(type_tag)) { - case bsoncxx::type::k_string: { + switch (static_cast(type_tag)) { + case milvus::bson::type::k_string: { int32_t len = *reinterpret_cast(ptr); ptr += 4 + len; break; } - case bsoncxx::type::k_int32: + case milvus::bson::type::k_int32: ptr += 4; break; - case bsoncxx::type::k_int64: + case milvus::bson::type::k_int64: ptr += 8; break; - case bsoncxx::type::k_double: + case milvus::bson::type::k_double: ptr += 8; break; - case bsoncxx::type::k_bool: + case milvus::bson::type::k_bool: ptr += 1; break; - case bsoncxx::type::k_array: - case bsoncxx::type::k_document: { + case milvus::bson::type::k_array: + case milvus::bson::type::k_document: { int32_t len = *reinterpret_cast(ptr); ptr += len; break; @@ -506,7 +467,7 @@ class BsonView { inline BsonRawField ParseBsonField(const uint8_t* bson_data, size_t offset) const { const uint8_t* ptr = bson_data + offset; - auto type_tag = static_cast(*ptr++); + auto type_tag = static_cast(*ptr++); const char* key_cstr = reinterpret_cast(ptr); size_t key_len = strlen(key_cstr); @@ -517,30 +478,30 @@ class BsonView { template static std::optional - GetValueFromElement(const bsoncxx::document::element& element) { + GetValueFromElement(const milvus::bson::element& element) { if constexpr (std::is_same_v) { - if (element.type() == bsoncxx::type::k_int32) { + if (element.type() == milvus::bson::type::k_int32) { return element.get_int32().value; } } else if constexpr (std::is_same_v) { - if (element.type() == bsoncxx::type::k_int64) { + if (element.type() == milvus::bson::type::k_int64) { return element.get_int64().value; } } else if constexpr (std::is_same_v) { - if (element.type() == bsoncxx::type::k_double) { + if (element.type() == milvus::bson::type::k_double) { return element.get_double().value; } } else if constexpr (std::is_same_v) { - if (element.type() == bsoncxx::type::k_bool) { + if (element.type() == milvus::bson::type::k_bool) { return element.get_bool().value; } } else if constexpr (std::is_same_v) { - if (element.type() == bsoncxx::type::k_string) { + if (element.type() == milvus::bson::type::k_string) { return std::string(element.get_string().value.data(), element.get_string().value.size()); } } else if constexpr (std::is_same_v) { - if (element.type() == bsoncxx::type::k_string) { + if (element.type() == milvus::bson::type::k_string) { return std::string_view(element.get_string().value.data(), element.get_string().value.size()); } @@ -550,27 +511,27 @@ class BsonView { template static std::optional - GetValueFromBsonView(const bsoncxx::types::bson_value::view& value_view) { + GetValueFromBsonView(const milvus::bson::value_view& value_view) { switch (value_view.type()) { - case bsoncxx::type::k_int32: + case milvus::bson::type::k_int32: if constexpr (std::is_same_v) return value_view.get_int32().value; break; - case bsoncxx::type::k_int64: + case milvus::bson::type::k_int64: if constexpr (std::is_same_v) return value_view.get_int64().value; if constexpr (std::is_same_v) return static_cast(value_view.get_int64().value); break; - case bsoncxx::type::k_double: + case milvus::bson::type::k_double: if constexpr (std::is_same_v) return value_view.get_double().value; break; - case bsoncxx::type::k_bool: + case milvus::bson::type::k_bool: if constexpr (std::is_same_v) return value_view.get_bool().value; break; - case bsoncxx::type::k_string: + case milvus::bson::type::k_string: if constexpr (std::is_same_v) { return std::string(value_view.get_string().value.data(), value_view.get_string().value.size()); @@ -580,8 +541,8 @@ class BsonView { value_view.get_string().value.size()); } break; - case bsoncxx::type::k_array: - if constexpr (std::is_same_v) { + case milvus::bson::type::k_array: + if constexpr (std::is_same_v) { return value_view.get_array().value; } break; @@ -595,4 +556,4 @@ class BsonView { size_t size_; }; -} // namespace milvus \ No newline at end of file +} // namespace milvus diff --git a/internal/core/src/exec/expression/JsonContainsExpr.cpp b/internal/core/src/exec/expression/JsonContainsExpr.cpp index cda67f5bde..3c5ac718e3 100644 --- a/internal/core/src/exec/expression/JsonContainsExpr.cpp +++ b/internal/core/src/exec/expression/JsonContainsExpr.cpp @@ -866,7 +866,7 @@ PhyJsonContainsFilterExpr::ExecJsonContainsArrayByStats() { for (const auto& sub_value : array.value()) { auto sub_array = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>(sub_value.get_value()); if (!sub_array.has_value()) continue; @@ -1657,7 +1657,8 @@ PhyJsonContainsFilterExpr::ExecJsonContainsAllWithDiffTypeByStats() { } case proto::plan::GenericValue::kArrayVal: { auto val = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>( + sub_value.get_value()); if (!val.has_value()) { continue; } @@ -1895,7 +1896,7 @@ PhyJsonContainsFilterExpr::ExecJsonContainsAllArrayByStats() { std::set exist_elements_index; for (const auto& sub_value : array.value()) { auto sub_array = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>(sub_value.get_value()); if (!sub_array.has_value()) continue; @@ -2216,7 +2217,8 @@ PhyJsonContainsFilterExpr::ExecJsonContainsWithDiffTypeByStats() { } case proto::plan::GenericValue::kArrayVal: { auto val = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>( + sub_value.get_value()); if (!val.has_value()) { continue; } diff --git a/internal/core/src/exec/expression/JsonContainsExpr.h b/internal/core/src/exec/expression/JsonContainsExpr.h index b8b57d1fa4..01690d19ea 100644 --- a/internal/core/src/exec/expression/JsonContainsExpr.h +++ b/internal/core/src/exec/expression/JsonContainsExpr.h @@ -60,7 +60,7 @@ class ShreddingArrayBsonContainsArrayExecutor { bool matched = false; for (const auto& sub_value : array_view.value()) { auto sub_array = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>(sub_value.get_value()); if (!sub_array.has_value()) continue; for (const auto& element : elements_) { @@ -108,7 +108,7 @@ class ShreddingArrayBsonContainsAllArrayExecutor { std::set exist_elements_index; for (const auto& sub_value : array_view.value()) { auto sub_array = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>(sub_value.get_value()); if (!sub_array.has_value()) continue; @@ -317,7 +317,8 @@ class ShreddingArrayBsonContainsAllWithDiffTypeExecutor { } case proto::plan::GenericValue::kArrayVal: { auto val = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>( + sub_value.get_value()); if (!val.has_value()) { continue; } @@ -419,7 +420,8 @@ class ShreddingArrayBsonContainsAnyWithDiffTypeExecutor { } case proto::plan::GenericValue::kArrayVal: { auto val = milvus::BsonView::GetValueFromBsonView< - bsoncxx::array::view>(sub_value.get_value()); + milvus::bson::array_view>( + sub_value.get_value()); if (val.has_value() && CompareTwoJsonArray(val.value(), element.array_val())) { diff --git a/internal/core/src/exec/expression/UnaryExpr.cpp b/internal/core/src/exec/expression/UnaryExpr.cpp index 9276763a2d..a33d4c918d 100644 --- a/internal/core/src/exec/expression/UnaryExpr.cpp +++ b/internal/core/src/exec/expression/UnaryExpr.cpp @@ -31,7 +31,7 @@ #include "boost/container/vector.hpp" #include "boost/cstdint.hpp" -#include "bsoncxx/array/view.hpp" +#include "common/bson_view.h" #include "common/Consts.h" #include "common/EasyAssert.h" #include "common/Json.h" @@ -1235,8 +1235,8 @@ PhyUnaryRangeFilterExpr::ExecRangeVisitorImplJsonByStats() { return; } auto sub_array = milvus::BsonView::GetNthElementInArray< - bsoncxx::array::view>(array_value.value().data(), - array_index); + milvus::bson::array_view>(array_value.value().data(), + array_index); if (!sub_array.has_value()) { res_view[row_id] = (op_type == proto::plan::OpType::NotEqual); diff --git a/internal/core/src/exec/expression/Utils.h b/internal/core/src/exec/expression/Utils.h index bfa9ea148f..d0ff02f42e 100644 --- a/internal/core/src/exec/expression/Utils.h +++ b/internal/core/src/exec/expression/Utils.h @@ -132,8 +132,8 @@ CompareTwoJsonArray(T arr1, const proto::plan::Array& arr2) { template <> inline bool -CompareTwoJsonArray(bsoncxx::array::view arr1, - const proto::plan::Array& arr2) { +CompareTwoJsonArray(milvus::bson::array_view arr1, + const proto::plan::Array& arr2) { size_t bson_array_length = std::distance(arr1.begin(), arr1.end()); if (arr2.array_size() != bson_array_length) { @@ -151,7 +151,7 @@ CompareTwoJsonArray(bsoncxx::array::view arr1, switch (proto_elem.val_case()) { case proto::plan::GenericValue::kBoolVal: { - if (bson_elem.type() != bsoncxx::type::k_bool) { + if (bson_elem.type() != milvus::bson::type::k_bool) { return false; } if (bson_elem.get_bool().value != proto_elem.bool_val()) { @@ -160,12 +160,12 @@ CompareTwoJsonArray(bsoncxx::array::view arr1, break; } case proto::plan::GenericValue::kInt64Val: { - if (bson_elem.type() == bsoncxx::type::k_int32) { + if (bson_elem.type() == milvus::bson::type::k_int32) { const int32_t val = bson_elem.get_int32().value; if (val != proto_elem.int64_val()) { return false; } - } else if (bson_elem.type() == bsoncxx::type::k_int64) { + } else if (bson_elem.type() == milvus::bson::type::k_int64) { const int64_t val = bson_elem.get_int64().value; if (val != proto_elem.int64_val()) { return false; @@ -178,13 +178,13 @@ CompareTwoJsonArray(bsoncxx::array::view arr1, case proto::plan::GenericValue::kFloatVal: { double bson_val; switch (bson_elem.type()) { - case bsoncxx::type::k_int32: + case milvus::bson::type::k_int32: bson_val = bson_elem.get_int32().value; break; - case bsoncxx::type::k_int64: + case milvus::bson::type::k_int64: bson_val = bson_elem.get_int64().value; break; - case bsoncxx::type::k_double: + case milvus::bson::type::k_double: bson_val = bson_elem.get_double().value; break; default: @@ -196,7 +196,7 @@ CompareTwoJsonArray(bsoncxx::array::view arr1, break; } case proto::plan::GenericValue::kStringVal: { - if (bson_elem.type() != bsoncxx::type::k_string) { + if (bson_elem.type() != milvus::bson::type::k_string) { return false; } auto bson_str_view = bson_elem.get_string().value; diff --git a/internal/core/src/index/InvertedIndexTantivy.cpp b/internal/core/src/index/InvertedIndexTantivy.cpp index 437b5b130f..4709403188 100644 --- a/internal/core/src/index/InvertedIndexTantivy.cpp +++ b/internal/core/src/index/InvertedIndexTantivy.cpp @@ -616,8 +616,7 @@ InvertedIndexTantivy::BuildWithRawDataForUT(size_t n, } } else { for (size_t i = 0; i < n; i++) { - wrapper_->template add_array_data( - arr[i].data(), arr[i].size(), i); + wrapper_->add_array_data(arr[i].data(), arr[i].size(), i); } } } else { @@ -628,7 +627,7 @@ InvertedIndexTantivy::BuildWithRawDataForUT(size_t n, // only used in ut. auto arr = static_cast*>(values); for (size_t i = 0; i < n; i++) { - wrapper_->template add_array_data_by_single_segment_writer( + wrapper_->add_array_data_by_single_segment_writer( arr[i].data(), arr[i].size()); } } else { @@ -752,13 +751,12 @@ InvertedIndexTantivy::build_index_for_array( } auto length = data->is_valid(i) ? array_column[i].length() : 0; if (!inverted_index_single_segment_) { - wrapper_->template add_array_data( - reinterpret_cast( - array_column[i].data()), - length, - offset++); + wrapper_->add_array_data(reinterpret_cast( + array_column[i].data()), + length, + offset++); } else { - wrapper_->template add_array_data_by_single_segment_writer( + wrapper_->add_array_data_by_single_segment_writer( reinterpret_cast( array_column[i].data()), length); @@ -792,11 +790,10 @@ InvertedIndexTantivy::build_index_for_array( } auto length = data->is_valid(i) ? output.size() : 0; if (!inverted_index_single_segment_) { - wrapper_->template add_array_data( - output.data(), length, offset++); + wrapper_->add_array_data(output.data(), length, offset++); } else { - wrapper_->template add_array_data_by_single_segment_writer( - output.data(), length); + wrapper_->add_array_data_by_single_segment_writer(output.data(), + length); } } } diff --git a/internal/core/src/index/json_stats/JsonKeyStats.cpp b/internal/core/src/index/json_stats/JsonKeyStats.cpp index b4337d8a5c..8889231ae1 100644 --- a/internal/core/src/index/json_stats/JsonKeyStats.cpp +++ b/internal/core/src/index/json_stats/JsonKeyStats.cpp @@ -30,9 +30,6 @@ #include "NamedType/underlying_functionalities.hpp" #include "arrow/api.h" #include "boost/filesystem/operations.hpp" -#include "bsoncxx/builder/basic/document.hpp" -#include "bsoncxx/document/value.hpp" -#include "bsoncxx/document/view.hpp" #include "cachinglayer/Manager.h" #include "cachinglayer/Translator.h" #include "common/Consts.h" @@ -516,10 +513,8 @@ JsonKeyStats::BuildKeyStatsForNullRow() { } // add null bson to shared column - bsoncxx::builder::basic::document null_doc; - auto null_bson = null_doc.extract(); - parquet_writer_->AppendSharedRow(null_bson.view().data(), - null_bson.view().length()); + BsonDocument null_doc; + parquet_writer_->AppendSharedRow(null_doc.data(), null_doc.length()); parquet_writer_->AddCurrentRow(); } @@ -595,12 +590,13 @@ JsonKeyStats::BuildKeyStatsForRow(const char* json_str, uint32_t row_id) { } } - bsoncxx::builder::basic::document final_doc; - BsonBuilder::ConvertDomToBson(root, final_doc); + BsonDocument final_doc; + BsonBuilder::ConvertDomToBson(root, final_doc.get()); // build inverted index for shared key // cache pairs of (key, row_id/offset) into memory // when all rows processed, build it into disk - auto key_offsets = BsonBuilder::ExtractBsonKeyOffsets(final_doc.view()); + auto key_offsets = BsonBuilder::ExtractBsonKeyOffsets(final_doc.data(), + final_doc.length()); for (const auto& [key, offset] : key_offsets) { LOG_TRACE( "add record to bson inverted index: {} with row_id: {} and offset: " @@ -612,8 +608,7 @@ JsonKeyStats::BuildKeyStatsForRow(const char* json_str, uint32_t row_id) { field_id_); bson_inverted_index_->AddRecord(key, row_id, offset); } - auto bson = final_doc.extract(); - parquet_writer_->AppendSharedRow(bson.view().data(), bson.view().length()); + parquet_writer_->AppendSharedRow(final_doc.data(), final_doc.length()); parquet_writer_->AddCurrentRow(); } diff --git a/internal/core/src/index/json_stats/bson_builder.cpp b/internal/core/src/index/json_stats/bson_builder.cpp index 6ae4cf1ca7..7b7d4065d0 100644 --- a/internal/core/src/index/json_stats/bson_builder.cpp +++ b/internal/core/src/index/json_stats/bson_builder.cpp @@ -14,30 +14,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include "common/FastMem.h" #include -#include #include #include -#include #include #include #include -#include -#include +#include #include -#include "bsoncxx/array/value.hpp" -#include "bsoncxx/array/view.hpp" -#include "bsoncxx/builder/basic/array.hpp" -#include "bsoncxx/builder/basic/document.hpp" -#include "bsoncxx/builder/basic/kvp.hpp" -#include "bsoncxx/document/value.hpp" -#include "bsoncxx/types.hpp" -#include "bsoncxx/types/bson_value/value.hpp" #include "common/EasyAssert.h" #include "common/Utils.h" +#include "common/bson_shim.h" #include "glog/logging.h" #include "index/json_stats/bson_builder.h" #include "log/Log.h" @@ -51,88 +42,74 @@ namespace milvus::index { namespace { -using bsoncxx::builder::basic::kvp; - +// Append a simdjson DOM element to a libbson document/array target under the +// given key (for arrays the key is the stringified index). void -AppendDomElementToBsonArray(simdjson::dom::element elem, - bsoncxx::builder::basic::array& out); - -void -AppendDomElementToBsonDocument(simdjson::dom::element elem, - std::string_view key, - bsoncxx::builder::basic::document& out) { +AppendJsonElementToBson(simdjson::dom::element elem, + bson_t* out, + const char* key, + int key_len) { using simdjson::dom::element_type; auto type = elem.type(); - if (type == element_type::STRING) { - std::string s(std::string_view(elem.get_string())); - out.append(kvp(std::string(key), std::move(s))); - } else if (type == element_type::INT64) { - out.append(kvp(std::string(key), int64_t(elem.get_int64()))); - } else if (type == element_type::UINT64) { - out.append(kvp(std::string(key), double(elem.get_uint64()))); - } else if (type == element_type::DOUBLE) { - out.append(kvp(std::string(key), elem.get_double())); - } else if (type == element_type::BOOL) { - out.append(kvp(std::string(key), bool(elem.get_bool()))); - } else if (type == element_type::NULL_VALUE) { - out.append(kvp(std::string(key), bsoncxx::types::b_null{})); - } else if (type == element_type::OBJECT) { - bsoncxx::builder::basic::document sub; - for (auto [k, v] : elem.get_object()) { - AppendDomElementToBsonDocument(v, k, sub); + switch (type) { + case element_type::STRING: { + auto sv = std::string_view(elem.get_string()); + bson_append_utf8( + out, key, key_len, sv.data(), static_cast(sv.size())); + break; } - out.append(kvp(std::string(key), sub.extract())); - } else if (type == element_type::ARRAY) { - bsoncxx::builder::basic::array subarr; - for (simdjson::dom::element v : elem.get_array()) { - AppendDomElementToBsonArray(v, subarr); + case element_type::INT64: + bson_append_int64(out, key, key_len, int64_t(elem.get_int64())); + break; + case element_type::UINT64: + bson_append_double(out, key, key_len, double(elem.get_uint64())); + break; + case element_type::DOUBLE: + bson_append_double(out, key, key_len, double(elem.get_double())); + break; + case element_type::BOOL: + bson_append_bool(out, key, key_len, bool(elem.get_bool())); + break; + case element_type::NULL_VALUE: + bson_append_null(out, key, key_len); + break; + case element_type::OBJECT: { + bson_t child; + bson_append_document_begin(out, key, key_len, &child); + for (auto [k, v] : elem.get_object()) { + AppendJsonElementToBson( + v, &child, k.data(), static_cast(k.size())); + } + bson_append_document_end(out, &child); + break; } - out.append(kvp(std::string(key), subarr.extract())); - } else { - out.append(kvp(std::string(key), bsoncxx::types::b_null{})); + case element_type::ARRAY: { + bson_t child; + bson_append_array_begin(out, key, key_len, &child); + uint32_t i = 0; + char buf[16]; + const char* idx_key = nullptr; + for (simdjson::dom::element v : elem.get_array()) { + size_t klen = + bson_uint32_to_string(i, &idx_key, buf, sizeof(buf)); + AppendJsonElementToBson( + v, &child, idx_key, static_cast(klen)); + i++; + } + bson_append_array_end(out, &child); + break; + } + default: + bson_append_null(out, key, key_len); + break; } } -void -AppendDomElementToBsonArray(simdjson::dom::element elem, - bsoncxx::builder::basic::array& out) { - using simdjson::dom::element_type; - - auto type = elem.type(); - if (type == element_type::STRING) { - out.append(std::string(std::string_view(elem.get_string()))); - } else if (type == element_type::INT64) { - out.append(int64_t(elem.get_int64())); - } else if (type == element_type::UINT64) { - out.append(double(elem.get_uint64())); - } else if (type == element_type::DOUBLE) { - out.append(elem.get_double()); - } else if (type == element_type::BOOL) { - out.append(bool(elem.get_bool())); - } else if (type == element_type::NULL_VALUE) { - out.append(bsoncxx::types::b_null{}); - } else if (type == element_type::OBJECT) { - bsoncxx::builder::basic::document sub; - for (auto [k, v] : elem.get_object()) { - AppendDomElementToBsonDocument(v, k, sub); - } - out.append(sub.extract()); - } else if (type == element_type::ARRAY) { - bsoncxx::builder::basic::array subarr; - for (simdjson::dom::element v : elem.get_array()) { - AppendDomElementToBsonArray(v, subarr); - } - out.append(subarr.extract()); - } else { - out.append(bsoncxx::types::b_null{}); - } -} } // namespace -// Parse a JSON array string with simdjson and build an owning BSON array value -bsoncxx::array::value -BuildBsonArrayFromJsonString(const std::string& json_array) { +std::vector +BuildBsonArrayBytesFromJsonString(const std::string& json_array) { simdjson::dom::parser parser; simdjson::dom::element root = parser.parse(json_array); if (root.type() != simdjson::dom::element_type::ARRAY) { @@ -141,18 +118,20 @@ BuildBsonArrayFromJsonString(const std::string& json_array) { json_array); } - bsoncxx::builder::basic::array out; + bson_t arr; + bson_init(&arr); + uint32_t i = 0; + char buf[16]; + const char* idx_key = nullptr; for (simdjson::dom::element elem : root.get_array()) { - AppendDomElementToBsonArray(elem, out); + size_t klen = bson_uint32_to_string(i, &idx_key, buf, sizeof(buf)); + AppendJsonElementToBson(elem, &arr, idx_key, static_cast(klen)); + i++; } - return out.extract(); -} - -std::vector -BuildBsonArrayBytesFromJsonString(const std::string& json_array) { - auto arr_value = BuildBsonArrayFromJsonString(json_array); - auto view = arr_value.view(); - return std::vector(view.data(), view.data() + view.length()); + std::vector out(bson_get_data(&arr), + bson_get_data(&arr) + arr.len); + bson_destroy(&arr); + return out; } void @@ -189,31 +168,46 @@ DomNode BsonBuilder::CreateValueNode(const std::string& value, JSONType type) { switch (type) { case JSONType::NONE: { - return DomNode(bsoncxx::types::b_null{}); + DomScalar s; + s.type = JSONType::NONE; + return DomNode(std::move(s)); } case JSONType::BOOL: { - bool b = (value == "true" || value == "1"); - return DomNode(bsoncxx::types::b_bool{b}); + DomScalar s; + s.type = JSONType::BOOL; + s.b = (value == "true" || value == "1"); + return DomNode(std::move(s)); } case JSONType::INT32: { - int32_t i = std::stoi(value); - return DomNode(bsoncxx::types::b_int32{i}); + DomScalar s; + s.type = JSONType::INT32; + s.i32 = std::stoi(value); + return DomNode(std::move(s)); } case JSONType::INT64: { - int64_t l = std::stoll(value); - return DomNode(bsoncxx::types::b_int64{l}); + DomScalar s; + s.type = JSONType::INT64; + s.i64 = std::stoll(value); + return DomNode(std::move(s)); } case JSONType::DOUBLE: { - double d = std::stod(value); - return DomNode(bsoncxx::types::b_double{d}); + DomScalar s; + s.type = JSONType::DOUBLE; + s.d = std::stod(value); + return DomNode(std::move(s)); } case JSONType::STRING: { - return DomNode(bsoncxx::types::b_string{value}); + DomScalar s; + s.type = JSONType::STRING; + s.str = value; + return DomNode(std::move(s)); } case JSONType::ARRAY: { try { - auto arr_value = BuildBsonArrayFromJsonString(value); - return DomNode(bsoncxx::types::b_array{arr_value.view()}); + DomScalar s; + s.type = JSONType::ARRAY; + s.arr_bytes = BuildBsonArrayBytesFromJsonString(value); + return DomNode(std::move(s)); } catch (const simdjson::simdjson_error& e) { ThrowInfo( ErrorCode::UnexpectedError, @@ -241,19 +235,56 @@ BsonBuilder::CreateValueNode(const std::string& value, JSONType type) { } void -BsonBuilder::ConvertDomToBson(const DomNode& node, - bsoncxx::builder::basic::document& builder) { +BsonBuilder::ConvertDomToBson(const DomNode& node, bson_t* builder) { for (const auto& [key, child] : node.document_children) { + const char* k = key.c_str(); + const int klen = static_cast(key.size()); switch (child.type) { case DomNode::Type::VALUE: { - builder.append(bsoncxx::builder::basic::kvp( - key, child.bson_value.value())); + const DomScalar& s = child.value.value(); + switch (s.type) { + case JSONType::NONE: + bson_append_null(builder, k, klen); + break; + case JSONType::BOOL: + bson_append_bool(builder, k, klen, s.b); + break; + case JSONType::INT32: + bson_append_int32(builder, k, klen, s.i32); + break; + case JSONType::INT64: + bson_append_int64(builder, k, klen, s.i64); + break; + case JSONType::DOUBLE: + bson_append_double(builder, k, klen, s.d); + break; + case JSONType::STRING: + bson_append_utf8(builder, + k, + klen, + s.str.data(), + static_cast(s.str.size())); + break; + case JSONType::ARRAY: { + bson_t arr; + if (bson_init_static( + &arr, s.arr_bytes.data(), s.arr_bytes.size())) { + bson_append_array(builder, k, klen, &arr); + } + break; + } + default: + ThrowInfo(ErrorCode::Unsupported, + "Unsupported scalar JSON type {}", + static_cast(s.type)); + } break; } case DomNode::Type::DOCUMENT: { - bsoncxx::builder::basic::document sub_doc; - ConvertDomToBson(child, sub_doc); - builder.append(bsoncxx::builder::basic::kvp(key, sub_doc)); + bson_t child_doc; + bson_append_document_begin(builder, k, klen, &child_doc); + ConvertDomToBson(child, &child_doc); + bson_append_document_end(builder, &child_doc); break; } default: { @@ -284,7 +315,7 @@ BsonBuilder::ExtractOffsetsRecursive( size_t key_offset = ptr - root_base_ptr; // read key type - auto type = static_cast(*ptr++); + auto type = static_cast(*ptr++); // read key auto key_name = reinterpret_cast(ptr); @@ -295,13 +326,13 @@ BsonBuilder::ExtractOffsetsRecursive( // do not record key offset pair for null value // because null value is not a valid key - if (type != bsoncxx::type::k_null) { + if (type != milvus::bson::type::k_null) { result.emplace_back(key_path, key_offset); } // handle value switch (type) { - case bsoncxx::type::k_document: { + case milvus::bson::type::k_document: { ExtractOffsetsRecursive(root_base_ptr, ptr, key_path, result); // skip sub doc uint32_t child_len; @@ -309,7 +340,7 @@ BsonBuilder::ExtractOffsetsRecursive( ptr += child_len; break; } - case bsoncxx::type::k_array: { + case milvus::bson::type::k_array: { // not parse array // skip sub doc uint32_t child_len; @@ -317,29 +348,29 @@ BsonBuilder::ExtractOffsetsRecursive( ptr += child_len; break; } - case bsoncxx::type::k_string: { + case milvus::bson::type::k_string: { uint32_t str_len; milvus::fastmem::FastMemcpy(&str_len, ptr, 4); ptr += 4 + str_len; break; } - case bsoncxx::type::k_int32: { + case milvus::bson::type::k_int32: { ptr += 4; break; } - case bsoncxx::type::k_int64: { + case milvus::bson::type::k_int64: { ptr += 8; break; } - case bsoncxx::type::k_double: { + case milvus::bson::type::k_double: { ptr += 8; break; } - case bsoncxx::type::k_bool: { + case milvus::bson::type::k_bool: { ptr += 1; break; } - case bsoncxx::type::k_null: { + case milvus::bson::type::k_null: { break; } default: { diff --git a/internal/core/src/index/json_stats/bson_builder.h b/internal/core/src/index/json_stats/bson_builder.h index 2d8ea15d99..716abc3280 100644 --- a/internal/core/src/index/json_stats/bson_builder.h +++ b/internal/core/src/index/json_stats/bson_builder.h @@ -15,39 +15,78 @@ // limitations under the License. #pragma once -#include -#include +#include #include #include -#include #include #include #include -#include #include #include -#include "bsoncxx/document/view.hpp" #include "common/protobuf_utils.h" #include "index/json_stats/utils.h" namespace milvus::index { +// An owned scalar (or pre-serialized array) value held by a DomNode. Replaces +// the former bsoncxx::types::bson_value::value: we keep the raw payload here and +// append it to a libbson document at conversion time. +struct DomScalar { + JSONType type{JSONType::NONE}; + bool b{false}; + int32_t i32{0}; + int64_t i64{0}; + double d{0.0}; + std::string str; // STRING payload + std::vector arr_bytes; // ARRAY payload: raw BSON array bytes +}; + class DomNode { public: enum class Type { DOCUMENT, VALUE }; Type type; std::map document_children; - std::optional bson_value; + std::optional value; DomNode(Type t = Type::DOCUMENT) : type(t) { } - DomNode(bsoncxx::types::bson_value::value v) - : type(Type::VALUE), bson_value(std::move(v)) { + DomNode(DomScalar v) : type(Type::VALUE), value(std::move(v)) { } }; +// RAII owner of a libbson document buffer, replacing the former +// bsoncxx::builder::basic::document. Exposes the serialized BSON bytes. +class BsonDocument { + public: + BsonDocument() { + bson_init(&bson_); + } + ~BsonDocument() { + bson_destroy(&bson_); + } + BsonDocument(const BsonDocument&) = delete; + BsonDocument& + operator=(const BsonDocument&) = delete; + + bson_t* + get() { + return &bson_; + } + const uint8_t* + data() const { + return bson_get_data(&bson_); + } + uint32_t + length() const { + return bson_.len; + } + + private: + bson_t bson_; +}; + // Parse a JSON array string and return a self-owned buffer containing the // BSON array bytes (length-prefixed, including terminator). The returned // vector's data() can be used directly with bson_view. @@ -66,8 +105,7 @@ class BsonBuilder { CreateValueNode(const std::string& value, JSONType type); static void - ConvertDomToBson(const DomNode& node, - bsoncxx::builder::basic::document& builder); + ConvertDomToBson(const DomNode& node, bson_t* builder); // helper function to recursively extract keys with offset static void @@ -77,15 +115,6 @@ class BsonBuilder { const std::string& current_path, std::vector>& result); - static std::vector> - ExtractBsonKeyOffsets(const bsoncxx::document::view& view) { - std::vector> result; - const uint8_t* raw_data = view.data(); - - ExtractOffsetsRecursive(raw_data, raw_data, "", result); - return result; - } - static std::vector> ExtractBsonKeyOffsets(const uint8_t* data, size_t size) { std::vector> result; diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt index 18cc19dbe6..3392ae42e9 100644 --- a/internal/core/thirdparty/CMakeLists.txt +++ b/internal/core/thirdparty/CMakeLists.txt @@ -35,7 +35,6 @@ add_subdirectory(boost_ext) add_subdirectory(rocksdb) add_subdirectory(rdkafka) add_subdirectory(simdjson) -add_subdirectory(bsoncxx) add_subdirectory(tantivy) if (LINUX) diff --git a/internal/core/thirdparty/bsoncxx/CMakeLists.txt b/internal/core/thirdparty/bsoncxx/CMakeLists.txt deleted file mode 100644 index e66d9e83db..0000000000 --- a/internal/core/thirdparty/bsoncxx/CMakeLists.txt +++ /dev/null @@ -1,63 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -#------------------------------------------------------------------------------- - -message(STATUS "Building mongo-cxx-driver (bsoncxx only) from source") - -# Disable tests and examples for mongo-cxx-driver -set(ENABLE_TESTS OFF CACHE INTERNAL "") -set(BUILD_TESTING OFF CACHE INTERNAL "") -set(ENABLE_EXAMPLES OFF CACHE INTERNAL "") -set(BUILD_MONGOCXX OFF CACHE INTERNAL "") -set(BUILD_SHARED_LIBS ON CACHE INTERNAL "") -set(BUILD_VERSION "0.0.0" CACHE INTERNAL "") - -# Disable optional features in mongo-c-driver to avoid linking issues on macOS -set(ENABLE_SASL OFF CACHE INTERNAL "") -set(ENABLE_SNAPPY OFF CACHE INTERNAL "") -set(ENABLE_ZSTD OFF CACHE INTERNAL "") -set(ENABLE_SSL OFF CACHE INTERNAL "") - -# Use FetchContent to download and build mongo-cxx-driver at configure time -FetchContent_Declare( - mongo-cxx-driver - GIT_REPOSITORY https://github.com/mongodb/mongo-cxx-driver.git - GIT_TAG releases/v3.11 - GIT_SHALLOW TRUE -) - -FetchContent_MakeAvailable(mongo-cxx-driver) - -# The targets are now available: bsoncxx_shared -# Create aliases for easier use -if(TARGET bsoncxx_shared) - add_library(bsoncxx ALIAS bsoncxx_shared) - message(STATUS "bsoncxx target created as alias to bsoncxx_shared") -endif() - -# Get source directories from FetchContent -FetchContent_GetProperties(mongo-cxx-driver SOURCE_DIR MONGO_CXX_SOURCE_DIR) - -# Set the include directories for use by other CMakeLists that use include_directories() -# We need: -# - v_noabi directory for the main headers -# - base include for v1 headers -# - build/lib for generated config headers -set(BSONCXX_INCLUDE_DIR - "${MONGO_CXX_SOURCE_DIR}/src/bsoncxx/include/bsoncxx/v_noabi" - "${MONGO_CXX_SOURCE_DIR}/src/bsoncxx/include" - "${CMAKE_BINARY_DIR}/3rdparty_download/mongo-cxx-driver-build/src/bsoncxx/lib/bsoncxx/v_noabi" - "${CMAKE_BINARY_DIR}/3rdparty_download/mongo-c-driver-src/src/libbson/src/bson" - "${CMAKE_BINARY_DIR}/3rdparty_download/mongo-c-driver-build/src/libbson/src/bson" - CACHE INTERNAL "Path to bsoncxx include directories") - -message("BSONCXX_INCLUDE_DIR: ${BSONCXX_INCLUDE_DIR}") diff --git a/internal/core/unittest/test_json_stats/test_bson_builder.cpp b/internal/core/unittest/test_json_stats/test_bson_builder.cpp index afc7a7d8de..875076740b 100644 --- a/internal/core/unittest/test_json_stats/test_bson_builder.cpp +++ b/internal/core/unittest/test_json_stats/test_bson_builder.cpp @@ -1,9 +1,4 @@ -#include -#include -#include -#include -#include -#include +#include #include #include #include @@ -19,12 +14,7 @@ #include #include -#include "bsoncxx/array/element.hpp" -#include "bsoncxx/array/view.hpp" -#include "bsoncxx/builder/basic/kvp.hpp" -#include "bsoncxx/document/element.hpp" -#include "bsoncxx/document/value.hpp" -#include "bsoncxx/document/view.hpp" +#include "common/bson_shim.h" #include "common/bson_view.h" #include "common/protobuf_utils.h" #include "gtest/gtest.h" @@ -44,7 +34,7 @@ TEST_F(DomNodeTest, DefaultConstructorTest) { DomNode node; EXPECT_EQ(node.type, DomNode::Type::DOCUMENT); EXPECT_TRUE(node.document_children.empty()); - EXPECT_FALSE(node.bson_value.has_value()); + EXPECT_FALSE(node.value.has_value()); } TEST_F(DomNodeTest, TypeConstructorTest) { @@ -52,37 +42,42 @@ TEST_F(DomNodeTest, TypeConstructorTest) { DomNode doc_node(DomNode::Type::DOCUMENT); EXPECT_EQ(doc_node.type, DomNode::Type::DOCUMENT); EXPECT_TRUE(doc_node.document_children.empty()); - EXPECT_FALSE(doc_node.bson_value.has_value()); + EXPECT_FALSE(doc_node.value.has_value()); // Test VALUE type DomNode value_node(DomNode::Type::VALUE); EXPECT_EQ(value_node.type, DomNode::Type::VALUE); EXPECT_TRUE(value_node.document_children.empty()); - EXPECT_FALSE(value_node.bson_value.has_value()); + EXPECT_FALSE(value_node.value.has_value()); } TEST_F(DomNodeTest, ValueConstructorTest) { // Test with boolean value - bsoncxx::types::b_bool bool_val{true}; - DomNode bool_node{bsoncxx::types::bson_value::value(bool_val)}; + DomScalar bool_scalar; + bool_scalar.type = JSONType::BOOL; + bool_scalar.b = true; + DomNode bool_node{std::move(bool_scalar)}; EXPECT_EQ(bool_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(bool_node.bson_value.has_value()); - EXPECT_TRUE(bool_node.bson_value.value().view().get_bool()); + EXPECT_TRUE(bool_node.value.has_value()); + EXPECT_TRUE(bool_node.value->b); // Test with int32 value - bsoncxx::types::b_int32 int_val{42}; - DomNode int_node{bsoncxx::types::bson_value::value(int_val)}; + DomScalar int_scalar; + int_scalar.type = JSONType::INT32; + int_scalar.i32 = 42; + DomNode int_node{std::move(int_scalar)}; EXPECT_EQ(int_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(int_node.bson_value.has_value()); - EXPECT_EQ(int_node.bson_value.value().view().get_int32(), 42); + EXPECT_TRUE(int_node.value.has_value()); + EXPECT_EQ(int_node.value->i32, 42); // Test with string value - bsoncxx::types::b_string str_val{"test"}; - DomNode str_node{bsoncxx::types::bson_value::value(str_val)}; + DomScalar str_scalar; + str_scalar.type = JSONType::STRING; + str_scalar.str = "test"; + DomNode str_node{std::move(str_scalar)}; EXPECT_EQ(str_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(str_node.bson_value.has_value()); - EXPECT_STREQ(str_node.bson_value.value().view().get_string().value.data(), - "test"); + EXPECT_TRUE(str_node.value.has_value()); + EXPECT_STREQ(str_node.value->str.c_str(), "test"); } TEST_F(DomNodeTest, DocumentChildrenTest) { @@ -97,17 +92,17 @@ TEST_F(DomNodeTest, DocumentChildrenTest) { EXPECT_EQ(root.document_children["child2"].type, DomNode::Type::DOCUMENT); // Test nested document - root.document_children["child2"].document_children["nested"] = DomNode( - bsoncxx::types::bson_value::value(bsoncxx::types::b_int32{123})); + DomScalar nested_scalar; + nested_scalar.type = JSONType::INT32; + nested_scalar.i32 = 123; + root.document_children["child2"].document_children["nested"] = + DomNode(std::move(nested_scalar)); EXPECT_EQ(root.document_children["child2"].document_children["nested"].type, DomNode::Type::VALUE); - EXPECT_EQ(root.document_children["child2"] - .document_children["nested"] - .bson_value.value() - .view() - .get_int32(), - 123); + EXPECT_EQ( + root.document_children["child2"].document_children["nested"].value->i32, + 123); } class BsonBuilderTest : public ::testing::Test { @@ -126,57 +121,52 @@ TEST_F(BsonBuilderTest, CreateValueNodeTest) { // Test NONE type auto none_node = builder.CreateValueNode("", JSONType::NONE); EXPECT_EQ(none_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(none_node.bson_value.value().view().type() == - bsoncxx::type::k_null); + EXPECT_TRUE(none_node.value->type == JSONType::NONE); // Test BOOL type auto true_node = builder.CreateValueNode("true", JSONType::BOOL); EXPECT_EQ(true_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(true_node.bson_value.value().view().type() == - bsoncxx::type::k_bool); - EXPECT_TRUE(true_node.bson_value.value().view().get_bool().value); + EXPECT_TRUE(true_node.value->type == JSONType::BOOL); + EXPECT_TRUE(true_node.value->b); auto false_node = builder.CreateValueNode("false", JSONType::BOOL); EXPECT_EQ(false_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(false_node.bson_value.value().view().type() == - bsoncxx::type::k_bool); - EXPECT_FALSE(false_node.bson_value.value().view().get_bool().value); + EXPECT_TRUE(false_node.value->type == JSONType::BOOL); + EXPECT_FALSE(false_node.value->b); // Test INT64 type auto int64_node = builder.CreateValueNode("9223372036854775807", JSONType::INT64); EXPECT_EQ(int64_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(int64_node.bson_value.value().view().type() == - bsoncxx::type::k_int64); - EXPECT_EQ(int64_node.bson_value.value().view().get_int64().value, - 9223372036854775807LL); + EXPECT_TRUE(int64_node.value->type == JSONType::INT64); + EXPECT_EQ(int64_node.value->i64, 9223372036854775807LL); // Test DOUBLE type auto double_node = builder.CreateValueNode("3.14159", JSONType::DOUBLE); EXPECT_EQ(double_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(double_node.bson_value.value().view().type() == - bsoncxx::type::k_double); - EXPECT_DOUBLE_EQ(double_node.bson_value.value().view().get_double().value, - 3.14159); + EXPECT_TRUE(double_node.value->type == JSONType::DOUBLE); + EXPECT_DOUBLE_EQ(double_node.value->d, 3.14159); // Test STRING type auto string_node = builder.CreateValueNode("hello world", JSONType::STRING); EXPECT_EQ(string_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(string_node.bson_value.value().view().type() == - bsoncxx::type::k_string); - EXPECT_EQ(string_node.bson_value.value().view().get_string().value, - "hello world"); + EXPECT_TRUE(string_node.value->type == JSONType::STRING); + EXPECT_EQ(string_node.value->str, "hello world"); // Test ARRAY type auto array_node = builder.CreateValueNode("[1, 2, 3]", JSONType::ARRAY); EXPECT_EQ(array_node.type, DomNode::Type::VALUE); - EXPECT_TRUE(array_node.bson_value.value().view().type() == - bsoncxx::type::k_array); - auto array_view = array_node.bson_value.value().view().get_array().value; + EXPECT_TRUE(array_node.value->type == JSONType::ARRAY); + // The array is stored as raw BSON array bytes; parse them back to verify. + milvus::bson::array_view array_view(array_node.value->arr_bytes.data(), + array_node.value->arr_bytes.size()); EXPECT_EQ(std::distance(array_view.begin(), array_view.end()), 3); - EXPECT_EQ(array_view[0].get_int64().value, 1); - EXPECT_EQ(array_view[1].get_int64().value, 2); - EXPECT_EQ(array_view[2].get_int64().value, 3); + auto ait = array_view.begin(); + EXPECT_EQ(ait->get_int64().value, 1); + ++ait; + EXPECT_EQ(ait->get_int64().value, 2); + ++ait; + EXPECT_EQ(ait->get_int64().value, 3); // Test invalid type EXPECT_THROW(builder.CreateValueNode("value", static_cast(999)), @@ -192,12 +182,7 @@ TEST_F(BsonBuilderTest, AppendToDomTest) { EXPECT_TRUE(root.document_children.find("key1") != root.document_children.end()); EXPECT_EQ(root.document_children["key1"].type, DomNode::Type::VALUE); - EXPECT_EQ(root.document_children["key1"] - .bson_value.value() - .view() - .get_string() - .value, - "value1"); + EXPECT_EQ(root.document_children["key1"].value->str, "value1"); // Test nested document append builder.AppendToDom( @@ -218,10 +203,7 @@ TEST_F(BsonBuilderTest, AppendToDomTest) { EXPECT_EQ(root.document_children["level1"] .document_children["level2"] .document_children["key2"] - .bson_value.value() - .view() - .get_int64() - .value, + .value->i64, 42); // Test overwriting existing value with document @@ -229,13 +211,9 @@ TEST_F(BsonBuilderTest, AppendToDomTest) { EXPECT_EQ(root.document_children["key1"].type, DomNode::Type::DOCUMENT); EXPECT_EQ(root.document_children["key1"].document_children["nested"].type, DomNode::Type::VALUE); - EXPECT_EQ(root.document_children["key1"] - .document_children["nested"] - .bson_value.value() - .view() - .get_string() - .value, - "value3"); + EXPECT_EQ( + root.document_children["key1"].document_children["nested"].value->str, + "value3"); } TEST_F(BsonBuilderTest, ConvertDomToBsonTest) { @@ -252,22 +230,44 @@ TEST_F(BsonBuilderTest, ConvertDomToBsonTest) { builder.AppendToDom(root, {"array_field"}, "[1, 2, 3]", JSONType::ARRAY); // Convert to BSON - bsoncxx::builder::basic::document bson_doc; - builder.ConvertDomToBson(root, bson_doc); - auto bson_view = bson_doc.view(); + BsonDocument bson_doc; + builder.ConvertDomToBson(root, bson_doc.get()); - // Verify the converted BSON document - EXPECT_EQ(bson_view["string_field"].get_string().value, "hello"); - EXPECT_EQ(bson_view["int_field"].get_int64().value, 42); - EXPECT_DOUBLE_EQ(bson_view["double_field"].get_double().value, 3.14); - EXPECT_TRUE(bson_view["bool_field"].get_bool().value); - EXPECT_EQ(bson_view["nested"]["field"].get_string().value, "nested_value"); + BsonView view(bson_doc.data(), bson_doc.length()); + milvus::bson::document_view doc_view(bson_doc.data(), bson_doc.length()); - auto array_view = bson_view["array_field"].get_array().value; + // Verify scalar fields via FindByPath + auto string_field = view.FindByPath(doc_view, {"string_field"}); + EXPECT_TRUE(string_field.has_value()); + EXPECT_EQ(string_field.value().get_string().value, "hello"); + + auto int_field = view.FindByPath(doc_view, {"int_field"}); + EXPECT_TRUE(int_field.has_value()); + EXPECT_EQ(int_field.value().get_int64().value, 42); + + auto double_field = view.FindByPath(doc_view, {"double_field"}); + EXPECT_TRUE(double_field.has_value()); + EXPECT_DOUBLE_EQ(double_field.value().get_double().value, 3.14); + + auto bool_field = view.FindByPath(doc_view, {"bool_field"}); + EXPECT_TRUE(bool_field.has_value()); + EXPECT_TRUE(bool_field.value().get_bool().value); + + auto nested_field = view.FindByPath(doc_view, {"nested", "field"}); + EXPECT_TRUE(nested_field.has_value()); + EXPECT_EQ(nested_field.value().get_string().value, "nested_value"); + + // Verify array + auto array_field = view.FindByPath(doc_view, {"array_field"}); + EXPECT_TRUE(array_field.has_value()); + auto array_view = array_field.value().get_array().value; EXPECT_EQ(std::distance(array_view.begin(), array_view.end()), 3); - EXPECT_EQ(array_view[0].get_int64().value, 1); - EXPECT_EQ(array_view[1].get_int64().value, 2); - EXPECT_EQ(array_view[2].get_int64().value, 3); + auto ait = array_view.begin(); + EXPECT_EQ(ait->get_int64().value, 1); + ++ait; + EXPECT_EQ(ait->get_int64().value, 2); + ++ait; + EXPECT_EQ(ait->get_int64().value, 3); } TEST_F(BsonBuilderTest, ComplexDocumentTest) { @@ -288,104 +288,158 @@ TEST_F(BsonBuilderTest, ComplexDocumentTest) { builder.AppendToDom(root, {"metadata", "count"}, "1000", JSONType::INT64); // Convert to BSON - bsoncxx::builder::basic::document bson_doc; - builder.ConvertDomToBson(root, bson_doc); - auto bson_view = bson_doc.view(); + BsonDocument bson_doc; + builder.ConvertDomToBson(root, bson_doc.get()); + + BsonView view(bson_doc.data(), bson_doc.length()); + milvus::bson::document_view doc_view(bson_doc.data(), bson_doc.length()); // Verify the complex document structure - EXPECT_EQ(bson_view["user"]["name"].get_string().value, "John"); - EXPECT_EQ(bson_view["user"]["age"].get_int64().value, 30); - EXPECT_TRUE(bson_view["user"]["active"].get_bool().value); - EXPECT_EQ(bson_view["user"]["address"]["city"].get_string().value, - "New York"); - EXPECT_EQ(bson_view["user"]["address"]["zip"].get_string().value, "10001"); - EXPECT_EQ(bson_view["metadata"]["version"].get_string().value, "1.0"); - EXPECT_EQ(bson_view["metadata"]["count"].get_int64().value, 1000); + auto name = view.FindByPath(doc_view, {"user", "name"}); + EXPECT_TRUE(name.has_value()); + EXPECT_EQ(name.value().get_string().value, "John"); + + auto age = view.FindByPath(doc_view, {"user", "age"}); + EXPECT_TRUE(age.has_value()); + EXPECT_EQ(age.value().get_int64().value, 30); + + auto active = view.FindByPath(doc_view, {"user", "active"}); + EXPECT_TRUE(active.has_value()); + EXPECT_TRUE(active.value().get_bool().value); + + auto city = view.FindByPath(doc_view, {"user", "address", "city"}); + EXPECT_TRUE(city.has_value()); + EXPECT_EQ(city.value().get_string().value, "New York"); + + auto zip = view.FindByPath(doc_view, {"user", "address", "zip"}); + EXPECT_TRUE(zip.has_value()); + EXPECT_EQ(zip.value().get_string().value, "10001"); + + auto version = view.FindByPath(doc_view, {"metadata", "version"}); + EXPECT_TRUE(version.has_value()); + EXPECT_EQ(version.value().get_string().value, "1.0"); + + auto count = view.FindByPath(doc_view, {"metadata", "count"}); + EXPECT_TRUE(count.has_value()); + EXPECT_EQ(count.value().get_int64().value, 1000); // Verify array - auto scores_array = bson_view["user"]["scores"].get_array().value; + auto scores = view.FindByPath(doc_view, {"user", "scores"}); + EXPECT_TRUE(scores.has_value()); + auto scores_array = scores.value().get_array().value; EXPECT_EQ(std::distance(scores_array.begin(), scores_array.end()), 3); - EXPECT_EQ(scores_array[0].get_int64().value, 85); - EXPECT_EQ(scores_array[1].get_int64().value, 90); - EXPECT_EQ(scores_array[2].get_int64().value, 95); + auto sit = scores_array.begin(); + EXPECT_EQ(sit->get_int64().value, 85); + ++sit; + EXPECT_EQ(sit->get_int64().value, 90); + ++sit; + EXPECT_EQ(sit->get_int64().value, 95); } TEST_F(BsonBuilderTest, ExtractBsonKeyOffsetsTest) { { - auto doc = bsoncxx::from_json(R"({ "age": 30 })"); - auto offsets = BsonBuilder::ExtractBsonKeyOffsets(doc.view()); + bson_t doc; + bson_init(&doc); + bson_append_int64(&doc, "age", -1, 30); + const uint8_t* data = bson_get_data(&doc); + uint32_t len = doc.len; + + auto offsets = BsonBuilder::ExtractBsonKeyOffsets(data, len); EXPECT_EQ(offsets.size(), 1); EXPECT_EQ(offsets[0].first, "/age"); EXPECT_EQ(offsets[0].second, 4); - BsonView view(doc.view().data(), doc.view().length()); + BsonView view(data, len); auto res = view.ParseAsValueAtOffset(4); EXPECT_EQ(res.value(), 30); + + bson_destroy(&doc); } { - auto doc = bsoncxx::from_json(R"({ "age": "30"})"); - auto offsets = BsonBuilder::ExtractBsonKeyOffsets(doc.view()); + bson_t doc; + bson_init(&doc); + bson_append_utf8(&doc, "age", -1, "30", -1); + const uint8_t* data = bson_get_data(&doc); + uint32_t len = doc.len; + + auto offsets = BsonBuilder::ExtractBsonKeyOffsets(data, len); EXPECT_EQ(offsets.size(), 1); EXPECT_EQ(offsets[0].first, "/age"); EXPECT_EQ(offsets[0].second, 4); - auto bson_view = BsonView(doc.view().data(), doc.view().length()); + BsonView bson_view(data, len); auto res = bson_view.ParseAsValueAtOffset(4); - EXPECT_STREQ(res.value().data(), "30"); + EXPECT_STREQ(std::string(res.value()).c_str(), "30"); + + bson_destroy(&doc); } { - auto doc = bsoncxx::from_json( - R"({ "age": 30, "name": "Alice", "active": true })"); - auto offsets = BsonBuilder::ExtractBsonKeyOffsets(doc.view()); + bson_t doc; + bson_init(&doc); + bson_append_int64(&doc, "age", -1, 30); + bson_append_utf8(&doc, "name", -1, "Alice", -1); + bson_append_bool(&doc, "active", -1, true); + const uint8_t* data = bson_get_data(&doc); + uint32_t len = doc.len; + + auto offsets = BsonBuilder::ExtractBsonKeyOffsets(data, len); EXPECT_EQ(offsets.size(), 3); EXPECT_EQ(offsets[0].first, "/age"); EXPECT_EQ(offsets[1].first, "/name"); EXPECT_EQ(offsets[2].first, "/active"); - auto view = BsonView(doc.view().data(), doc.view().length()); + BsonView view(data, len); auto res1 = view.ParseAsValueAtOffset(offsets[0].second); EXPECT_EQ(res1.value(), 30); auto res2 = view.ParseAsValueAtOffset(offsets[1].second); - EXPECT_STREQ(res2.value().data(), "Alice"); + EXPECT_STREQ(std::string(res2.value()).c_str(), "Alice"); auto res3 = view.ParseAsValueAtOffset(offsets[2].second); EXPECT_EQ(res3.value(), true); + + bson_destroy(&doc); } } TEST_F(BsonBuilderTest, ExtractBsonKeyOffsetsTestAllTypes) { // Create a complex BSON document with nested structures - bsoncxx::builder::basic::document doc; + bson_t doc; + bson_init(&doc); // Add simple fields - doc.append(bsoncxx::builder::basic::kvp("string_field", "value1")); - doc.append(bsoncxx::builder::basic::kvp("int_field", 42)); - doc.append(bsoncxx::builder::basic::kvp("double_field", 3.14)); - doc.append(bsoncxx::builder::basic::kvp("bool_field", true)); + bson_append_utf8(&doc, "string_field", -1, "value1", -1); + bson_append_int32(&doc, "int_field", -1, 42); + bson_append_double(&doc, "double_field", -1, 3.14); + bson_append_bool(&doc, "bool_field", -1, true); // Add nested document - bsoncxx::builder::basic::document nested_doc; - nested_doc.append( - bsoncxx::builder::basic::kvp("nested_string", "nested_value")); - nested_doc.append(bsoncxx::builder::basic::kvp("nested_int", 123)); - doc.append(bsoncxx::builder::basic::kvp("nested_doc", nested_doc)); + bson_t nested_doc; + bson_append_document_begin(&doc, "nested_doc", -1, &nested_doc); + bson_append_utf8(&nested_doc, "nested_string", -1, "nested_value", -1); + bson_append_int32(&nested_doc, "nested_int", -1, 123); + bson_append_document_end(&doc, &nested_doc); // Add array - bsoncxx::builder::basic::array arr; - arr.append("array_item1"); - arr.append(456); - arr.append(false); - doc.append(bsoncxx::builder::basic::kvp("array_field", arr)); + bson_t arr; + bson_append_array_begin(&doc, "array_field", -1, &arr); + bson_append_utf8(&arr, "0", -1, "array_item1", -1); + bson_append_int32(&arr, "1", -1, 456); + bson_append_bool(&arr, "2", -1, false); + bson_append_array_end(&doc, &arr); // Add nested array with document - bsoncxx::builder::basic::array nested_arr; - bsoncxx::builder::basic::document arr_doc; - arr_doc.append( - bsoncxx::builder::basic::kvp("arr_doc_field", "arr_doc_value")); - nested_arr.append(arr_doc); - doc.append(bsoncxx::builder::basic::kvp("nested_array", nested_arr)); + bson_t nested_arr; + bson_append_array_begin(&doc, "nested_array", -1, &nested_arr); + bson_t arr_doc; + bson_append_document_begin(&nested_arr, "0", -1, &arr_doc); + bson_append_utf8(&arr_doc, "arr_doc_field", -1, "arr_doc_value", -1); + bson_append_document_end(&nested_arr, &arr_doc); + bson_append_array_end(&doc, &nested_arr); + + const uint8_t* data = bson_get_data(&doc); + uint32_t doc_len = doc.len; // Extract offsets - auto offsets = BsonBuilder::ExtractBsonKeyOffsets(doc.view()); + auto offsets = BsonBuilder::ExtractBsonKeyOffsets(data, doc_len); // Verify results EXPECT_FALSE(offsets.empty()); @@ -409,7 +463,7 @@ TEST_F(BsonBuilderTest, ExtractBsonKeyOffsetsTestAllTypes) { EXPECT_TRUE(offset_map.find("/nested_array") != offset_map.end()); // Verify offsets are within document bounds - size_t doc_size = doc.view().length(); + size_t doc_size = doc_len; for (const auto& [key, offset] : offsets) { EXPECT_LT(offset, doc_size) << "Offset for key " << key << " is out of bounds"; @@ -435,63 +489,79 @@ TEST_F(BsonBuilderTest, ExtractBsonKeyOffsetsTestAllTypes) { // 9. nested_array // 10. nested_array/0/arr_doc_field EXPECT_EQ(offsets.size(), 9); + + bson_destroy(&doc); } TEST_F(BsonBuilderTest, ExtractOffsetsRecursiveTest) { // Test empty document - bsoncxx::builder::basic::document empty_doc; + bson_t empty_doc; + bson_init(&empty_doc); + const uint8_t* empty_data = bson_get_data(&empty_doc); std::vector> empty_result; - builder_->ExtractOffsetsRecursive( - empty_doc.view().data(), empty_doc.view().data(), "", empty_result); + builder_->ExtractOffsetsRecursive(empty_data, empty_data, "", empty_result); EXPECT_TRUE(empty_result.empty()); + bson_destroy(&empty_doc); // Test simple document with one field - bsoncxx::builder::basic::document simple_doc; - simple_doc.append(bsoncxx::builder::basic::kvp("field", "value")); + bson_t simple_doc; + bson_init(&simple_doc); + bson_append_utf8(&simple_doc, "field", -1, "value", -1); + const uint8_t* simple_data = bson_get_data(&simple_doc); std::vector> simple_result; builder_->ExtractOffsetsRecursive( - simple_doc.view().data(), simple_doc.view().data(), "", simple_result); + simple_data, simple_data, "", simple_result); EXPECT_EQ(simple_result.size(), 1); EXPECT_EQ(simple_result[0].first, "/field"); + bson_destroy(&simple_doc); } TEST_F(BsonBuilderTest, ExtractOffsetsRecursiveWithNestedPathTest) { - bsoncxx::builder::basic::document doc; - bsoncxx::builder::basic::document nested_doc; - nested_doc.append(bsoncxx::builder::basic::kvp("nested_field", "value")); - doc.append(bsoncxx::builder::basic::kvp("parent", nested_doc)); + bson_t doc; + bson_init(&doc); + bson_t nested_doc; + bson_append_document_begin(&doc, "parent", -1, &nested_doc); + bson_append_utf8(&nested_doc, "nested_field", -1, "value", -1); + bson_append_document_end(&doc, &nested_doc); + const uint8_t* data = bson_get_data(&doc); std::vector> result; - builder_->ExtractOffsetsRecursive( - doc.view().data(), doc.view().data(), "prefix", result); + builder_->ExtractOffsetsRecursive(data, data, "prefix", result); EXPECT_EQ(result.size(), 2); EXPECT_EQ(result[0].first, "prefix/parent"); EXPECT_EQ(result[1].first, "prefix/parent/nested_field"); + + bson_destroy(&doc); } TEST_F(BsonBuilderTest, ExtractOffsetsRecursiveWithArrayElementsTest) { - bsoncxx::builder::basic::document doc; - bsoncxx::builder::basic::array arr; + bson_t doc; + bson_init(&doc); + bson_t arr; + bson_append_array_begin(&doc, "mixed_array", -1, &arr); // Add array with mixed types - arr.append("string_item"); - arr.append(42); - arr.append(true); + bson_append_utf8(&arr, "0", -1, "string_item", -1); + bson_append_int32(&arr, "1", -1, 42); + bson_append_bool(&arr, "2", -1, true); // Add nested document in array - bsoncxx::builder::basic::document nested_doc; - nested_doc.append(bsoncxx::builder::basic::kvp("nested_field", "value")); - arr.append(nested_doc); + bson_t nested_doc; + bson_append_document_begin(&arr, "3", -1, &nested_doc); + bson_append_utf8(&nested_doc, "nested_field", -1, "value", -1); + bson_append_document_end(&arr, &nested_doc); - doc.append(bsoncxx::builder::basic::kvp("mixed_array", arr)); + bson_append_array_end(&doc, &arr); + const uint8_t* data = bson_get_data(&doc); std::vector> result; - builder_->ExtractOffsetsRecursive( - doc.view().data(), doc.view().data(), "", result); + builder_->ExtractOffsetsRecursive(data, data, "", result); EXPECT_EQ(result.size(), 1); // mixed_array EXPECT_EQ(result[0].first, "/mixed_array"); + + bson_destroy(&doc); } } // namespace milvus::index diff --git a/internal/core/unittest/test_json_stats/test_bson_view.cpp b/internal/core/unittest/test_json_stats/test_bson_view.cpp index 3087eff913..80b2b77637 100644 --- a/internal/core/unittest/test_json_stats/test_bson_view.cpp +++ b/internal/core/unittest/test_json_stats/test_bson_view.cpp @@ -1,7 +1,4 @@ -#include -#include -#include -#include +#include #include #include #include @@ -15,11 +12,7 @@ #include #include -#include "bsoncxx/array/element.hpp" -#include "bsoncxx/array/view.hpp" -#include "bsoncxx/builder/basic/kvp.hpp" -#include "bsoncxx/document/element.hpp" -#include "bsoncxx/document/view.hpp" +#include "common/bson_shim.h" #include "common/bson_view.h" #include "common/protobuf_utils.h" #include "gtest/gtest.h" @@ -44,15 +37,18 @@ TEST_F(BsonViewTest, ConstructorTest) { TEST_F(BsonViewTest, ParseAsValueAtOffsetTest) { // Create a BSON document with various types - bsoncxx::builder::basic::document doc; - doc.append(bsoncxx::builder::basic::kvp("int32_field", 42)); - doc.append(bsoncxx::builder::basic::kvp("double_field", 3.14159)); - doc.append(bsoncxx::builder::basic::kvp("bool_field", true)); - doc.append(bsoncxx::builder::basic::kvp("string_field", "test string")); + bson_t doc; + bson_init(&doc); + bson_append_int32(&doc, "int32_field", -1, 42); + bson_append_double(&doc, "double_field", -1, 3.14159); + bson_append_bool(&doc, "bool_field", -1, true); + bson_append_utf8(&doc, "string_field", -1, "test string", -1); - BsonView view(doc.view().data(), doc.view().length()); + const uint8_t* data = bson_get_data(&doc); + uint32_t len = doc.len; + BsonView view(data, len); - auto offsets = BsonBuilder::ExtractBsonKeyOffsets(doc.view()); + auto offsets = BsonBuilder::ExtractBsonKeyOffsets(data, len); for (const auto& offset : offsets) { std::cout << offset.first << " " << offset.second << std::endl; } @@ -76,17 +72,20 @@ TEST_F(BsonViewTest, ParseAsValueAtOffsetTest) { auto string_val = view.ParseAsValueAtOffset(offsets[3].second); EXPECT_TRUE(string_val.has_value()); EXPECT_EQ(string_val.value(), "test string"); + + bson_destroy(&doc); } TEST_F(BsonViewTest, ParseAsArrayAtOffsetTest) { // Test case 1: offset = 0 (whole array) { - bsoncxx::builder::basic::array arr; - arr.append("value1"); - arr.append(42); - arr.append(true); + bson_t arr; + bson_init(&arr); + bson_append_utf8(&arr, "0", -1, "value1", -1); + bson_append_int32(&arr, "1", -1, 42); + bson_append_bool(&arr, "2", -1, true); - BsonView view(arr.view().data(), arr.view().length()); + BsonView view(bson_get_data(&arr), arr.len); auto array_view = view.ParseAsArrayAtOffset(0); // Verify array content @@ -94,26 +93,35 @@ TEST_F(BsonViewTest, ParseAsArrayAtOffsetTest) { EXPECT_EQ( std::distance(array_view.value().begin(), array_view.value().end()), 3); - EXPECT_STREQ(array_view.value()[0].get_string().value.data(), "value1"); - EXPECT_EQ(array_view.value()[1].get_int32(), 42); - EXPECT_TRUE(array_view.value()[2].get_bool()); + + auto it = array_view.value().begin(); + EXPECT_STREQ(std::string(it->get_string().value).c_str(), "value1"); + ++it; + EXPECT_EQ(it->get_int32().value, 42); + ++it; + EXPECT_TRUE(it->get_bool().value); + + bson_destroy(&arr); } // Test case 2: offset != 0 (array in document) { - bsoncxx::builder::basic::array arr; - arr.append("array_value1"); - arr.append(123); - arr.append(3.14159); + bson_t doc; + bson_init(&doc); + bson_append_utf8(&doc, "top_field", -1, "top_value", -1); + bson_t arr; + bson_append_array_begin(&doc, "array_field", -1, &arr); + bson_append_utf8(&arr, "0", -1, "array_value1", -1); + bson_append_int32(&arr, "1", -1, 123); + bson_append_double(&arr, "2", -1, 3.14159); + bson_append_array_end(&doc, &arr); - bsoncxx::builder::basic::document doc; - doc.append(bsoncxx::builder::basic::kvp("top_field", "top_value")); - doc.append(bsoncxx::builder::basic::kvp("array_field", arr)); - - BsonView view(doc.view().data(), doc.view().length()); + const uint8_t* data = bson_get_data(&doc); + uint32_t len = doc.len; + BsonView view(data, len); // Find the offset of the array - auto offsets = BsonBuilder::ExtractBsonKeyOffsets(doc.view()); + auto offsets = BsonBuilder::ExtractBsonKeyOffsets(data, len); size_t array_offset = 0; for (const auto& offset : offsets) { if (offset.first == "/array_field") { @@ -128,202 +136,249 @@ TEST_F(BsonViewTest, ParseAsArrayAtOffsetTest) { EXPECT_EQ( std::distance(array_view.value().begin(), array_view.value().end()), 3); - EXPECT_STREQ(array_view.value()[0].get_string().value.data(), + + auto it = array_view.value().begin(); + EXPECT_STREQ(std::string(it->get_string().value).c_str(), "array_value1"); - EXPECT_EQ(array_view.value()[1].get_int32(), 123); - EXPECT_DOUBLE_EQ(array_view.value()[2].get_double(), 3.14159); + ++it; + EXPECT_EQ(it->get_int32().value, 123); + ++it; + EXPECT_DOUBLE_EQ(it->get_double().value, 3.14159); + + bson_destroy(&doc); } // Test case 4: Error cases { - bsoncxx::builder::basic::document doc; - doc.append(bsoncxx::builder::basic::kvp("field", "value")); - BsonView view(doc.view().data(), doc.view().length()); + bson_t doc; + bson_init(&doc); + bson_append_utf8(&doc, "field", -1, "value", -1); + BsonView view(bson_get_data(&doc), doc.len); // Test invalid offset (out of range) EXPECT_THROW(view.ParseAsArrayAtOffset(1000), std::runtime_error); + + bson_destroy(&doc); } } TEST_F(BsonViewTest, FindByPathTest) { // Create a complex nested BSON document - bsoncxx::builder::basic::document nested_doc; - nested_doc.append(bsoncxx::builder::basic::kvp("nested_field", "value")); + bson_t doc; + bson_init(&doc); + bson_t nested_doc; + bson_append_document_begin(&doc, "level1", -1, &nested_doc); + bson_append_utf8(&nested_doc, "nested_field", -1, "value", -1); + bson_append_document_end(&doc, &nested_doc); + bson_append_utf8(&doc, "simple_field", -1, "simple_value", -1); - bsoncxx::builder::basic::document doc; - doc.append(bsoncxx::builder::basic::kvp("level1", nested_doc)); - doc.append(bsoncxx::builder::basic::kvp("simple_field", "simple_value")); - - BsonView view(doc.view().data(), doc.view().length()); + const uint8_t* data = bson_get_data(&doc); + uint32_t len = doc.len; + BsonView view(data, len); + milvus::bson::document_view doc_view(data, len); // Test finding nested field std::vector path = {"level1", "nested_field"}; - auto value = view.FindByPath(doc.view(), path); + auto value = view.FindByPath(doc_view, path); EXPECT_TRUE(value.has_value()); - EXPECT_STREQ(value.value().get_string().value.data(), "value"); + EXPECT_STREQ(std::string(value.value().get_string().value).c_str(), + "value"); // Test finding simple field path = {"simple_field"}; - value = view.FindByPath(doc.view(), path); + value = view.FindByPath(doc_view, path); EXPECT_TRUE(value.has_value()); - EXPECT_STREQ(value.value().get_string().value.data(), "simple_value"); + EXPECT_STREQ(std::string(value.value().get_string().value).c_str(), + "simple_value"); // Test non-existent path path = {"non_existent"}; - value = view.FindByPath(doc.view(), path); + value = view.FindByPath(doc_view, path); EXPECT_FALSE(value.has_value()); // Test invalid nested path path = {"level1", "non_existent"}; - value = view.FindByPath(doc.view(), path); + value = view.FindByPath(doc_view, path); EXPECT_FALSE(value.has_value()); + + bson_destroy(&doc); } TEST_F(BsonViewTest, GetNthElementInArrayTest) { // Create a BSON array with mixed types - bsoncxx::builder::basic::array arr; - arr.append("string_item"); - arr.append(42); - arr.append(true); - arr.append(3.14159); + bson_t arr; + bson_init(&arr); + bson_append_utf8(&arr, "0", -1, "string_item", -1); + bson_append_int32(&arr, "1", -1, 42); + bson_append_bool(&arr, "2", -1, true); + bson_append_double(&arr, "3", -1, 3.14159); + + const uint8_t* data = bson_get_data(&arr); // Test string element - auto string_val = - BsonView::GetNthElementInArray(arr.view().data(), 0); + auto string_val = BsonView::GetNthElementInArray(data, 0); EXPECT_TRUE(string_val.has_value()); EXPECT_EQ(string_val.value(), "string_item"); // Test int32 element - auto int_val = - BsonView::GetNthElementInArray(arr.view().data(), 1); + auto int_val = BsonView::GetNthElementInArray(data, 1); EXPECT_TRUE(int_val.has_value()); EXPECT_EQ(int_val.value(), 42); // Test bool element - auto bool_val = BsonView::GetNthElementInArray(arr.view().data(), 2); + auto bool_val = BsonView::GetNthElementInArray(data, 2); EXPECT_TRUE(bool_val.has_value()); EXPECT_TRUE(bool_val.value()); // Test double element - auto double_val = - BsonView::GetNthElementInArray(arr.view().data(), 3); + auto double_val = BsonView::GetNthElementInArray(data, 3); EXPECT_TRUE(double_val.has_value()); EXPECT_DOUBLE_EQ(double_val.value(), 3.14159); // Test out of bounds - auto out_of_bounds = - BsonView::GetNthElementInArray(arr.view().data(), 10); + auto out_of_bounds = BsonView::GetNthElementInArray(data, 10); EXPECT_FALSE(out_of_bounds.has_value()); // Test invalid type conversion - auto invalid_type = - BsonView::GetNthElementInArray(arr.view().data(), 1); + auto invalid_type = BsonView::GetNthElementInArray(data, 1); EXPECT_FALSE(invalid_type.has_value()); + + bson_destroy(&arr); } TEST_F(BsonViewTest, ParseBsonFieldTest) { // Create a simple BSON document - bsoncxx::builder::basic::document doc; - doc.append(bsoncxx::builder::basic::kvp("test_field", "test_value")); + bson_t doc; + bson_init(&doc); + bson_append_utf8(&doc, "test_field", -1, "test_value", -1); - BsonView view(doc.view().data(), doc.view().length()); + const uint8_t* data = bson_get_data(&doc); + BsonView view(data, doc.len); // Test field parsing - auto field = view.ParseBsonField(doc.view().data(), 4); - EXPECT_EQ(field.type, bsoncxx::type::k_string); + auto field = view.ParseBsonField(data, 4); + EXPECT_EQ(field.type, milvus::bson::type::k_string); EXPECT_EQ(field.key, "test_field"); EXPECT_NE(field.value_ptr, nullptr); + + bson_destroy(&doc); } TEST_F(BsonViewTest, GetValueFromElementTest) { // Create a BSON document with various types - bsoncxx::builder::basic::document doc; - doc.append(bsoncxx::builder::basic::kvp("int32_field", 42)); - doc.append(bsoncxx::builder::basic::kvp("double_field", 3.14159)); - doc.append(bsoncxx::builder::basic::kvp("bool_field", true)); - doc.append(bsoncxx::builder::basic::kvp("string_field", "test string")); + bson_t doc; + bson_init(&doc); + bson_append_int32(&doc, "int32_field", -1, 42); + bson_append_double(&doc, "double_field", -1, 3.14159); + bson_append_bool(&doc, "bool_field", -1, true); + bson_append_utf8(&doc, "string_field", -1, "test string", -1); - auto view = doc.view(); + milvus::bson::document_view view(bson_get_data(&doc), doc.len); + + // Collect elements by key for lookup + auto find_elem = [&](std::string_view key) -> milvus::bson::element { + for (auto&& e : view) { + if (e.key() == key) { + return e; + } + } + return milvus::bson::element(); + }; // Test int32 auto int32_val = - BsonView::GetValueFromElement(view["int32_field"]); + BsonView::GetValueFromElement(find_elem("int32_field")); EXPECT_TRUE(int32_val.has_value()); EXPECT_EQ(int32_val.value(), 42); // Test double auto double_val = - BsonView::GetValueFromElement(view["double_field"]); - EXPECT_TRUE(double_val.has_value()); - EXPECT_DOUBLE_EQ(double_val.value(), 3.14159); - - // Test bool - auto bool_val = BsonView::GetValueFromElement(view["bool_field"]); - EXPECT_TRUE(bool_val.has_value()); - EXPECT_TRUE(bool_val.value()); - - // Test string - auto string_val = - BsonView::GetValueFromElement(view["string_field"]); - EXPECT_TRUE(string_val.has_value()); - EXPECT_EQ(string_val.value(), "test string"); - - // Test string_view - auto string_view_val = - BsonView::GetValueFromElement(view["string_field"]); - EXPECT_TRUE(string_view_val.has_value()); - EXPECT_EQ(string_view_val.value(), "test string"); - - // Test invalid type conversion - auto invalid_val = - BsonView::GetValueFromElement(view["int32_field"]); - EXPECT_FALSE(invalid_val.has_value()); -} - -TEST_F(BsonViewTest, GetValueFromBsonViewTest) { - // Create a BSON document with various types - bsoncxx::builder::basic::document doc; - doc.append(bsoncxx::builder::basic::kvp("int32_field", 42)); - doc.append(bsoncxx::builder::basic::kvp("double_field", 3.14159)); - doc.append(bsoncxx::builder::basic::kvp("bool_field", true)); - doc.append(bsoncxx::builder::basic::kvp("string_field", "test string")); - - auto view = doc.view(); - - // Test int32 - auto int32_val = BsonView::GetValueFromBsonView( - view["int32_field"].get_value()); - EXPECT_TRUE(int32_val.has_value()); - EXPECT_EQ(int32_val.value(), 42); - - // Test double - auto double_val = BsonView::GetValueFromBsonView( - view["double_field"].get_value()); + BsonView::GetValueFromElement(find_elem("double_field")); EXPECT_TRUE(double_val.has_value()); EXPECT_DOUBLE_EQ(double_val.value(), 3.14159); // Test bool auto bool_val = - BsonView::GetValueFromBsonView(view["bool_field"].get_value()); + BsonView::GetValueFromElement(find_elem("bool_field")); EXPECT_TRUE(bool_val.has_value()); EXPECT_TRUE(bool_val.value()); // Test string - auto string_val = BsonView::GetValueFromBsonView( - view["string_field"].get_value()); + auto string_val = + BsonView::GetValueFromElement(find_elem("string_field")); + EXPECT_TRUE(string_val.has_value()); + EXPECT_EQ(string_val.value(), "test string"); + + // Test string_view + auto string_view_val = BsonView::GetValueFromElement( + find_elem("string_field")); + EXPECT_TRUE(string_view_val.has_value()); + EXPECT_EQ(string_view_val.value(), "test string"); + + // Test invalid type conversion + auto invalid_val = + BsonView::GetValueFromElement(find_elem("int32_field")); + EXPECT_FALSE(invalid_val.has_value()); + + bson_destroy(&doc); +} + +TEST_F(BsonViewTest, GetValueFromBsonViewTest) { + // Create a BSON document with various types + bson_t doc; + bson_init(&doc); + bson_append_int32(&doc, "int32_field", -1, 42); + bson_append_double(&doc, "double_field", -1, 3.14159); + bson_append_bool(&doc, "bool_field", -1, true); + bson_append_utf8(&doc, "string_field", -1, "test string", -1); + + milvus::bson::document_view view(bson_get_data(&doc), doc.len); + + auto find_value = [&](std::string_view key) -> milvus::bson::value_view { + for (auto&& e : view) { + if (e.key() == key) { + return e.get_value(); + } + } + return milvus::bson::value_view(); + }; + + // Test int32 + auto int32_val = + BsonView::GetValueFromBsonView(find_value("int32_field")); + EXPECT_TRUE(int32_val.has_value()); + EXPECT_EQ(int32_val.value(), 42); + + // Test double + auto double_val = + BsonView::GetValueFromBsonView(find_value("double_field")); + EXPECT_TRUE(double_val.has_value()); + EXPECT_DOUBLE_EQ(double_val.value(), 3.14159); + + // Test bool + auto bool_val = + BsonView::GetValueFromBsonView(find_value("bool_field")); + EXPECT_TRUE(bool_val.has_value()); + EXPECT_TRUE(bool_val.value()); + + // Test string + auto string_val = + BsonView::GetValueFromBsonView(find_value("string_field")); EXPECT_TRUE(string_val.has_value()); EXPECT_EQ(string_val.value(), "test string"); // Test string_view auto string_view_val = BsonView::GetValueFromBsonView( - view["string_field"].get_value()); + find_value("string_field")); EXPECT_TRUE(string_view_val.has_value()); EXPECT_EQ(string_view_val.value(), "test string"); // Test invalid type conversion - auto invalid_val = BsonView::GetValueFromBsonView( - view["int32_field"].get_value()); + auto invalid_val = + BsonView::GetValueFromBsonView(find_value("int32_field")); EXPECT_FALSE(invalid_val.has_value()); + + bson_destroy(&doc); } -} // namespace milvus::index \ No newline at end of file +} // namespace milvus::index diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh index 7618ed5b0f..eaa7de30e0 100644 --- a/scripts/3rdparty_build.sh +++ b/scripts/3rdparty_build.sh @@ -141,7 +141,10 @@ if [[ "$(uname -s)" == "Darwin" ]]; then arm_crypto_flags="" if [[ "$(uname -m)" == "arm64" ]]; then # Homebrew LLVM needs ARM crypto enabled for libsodium armcrypto sources on macOS arm64. - arm_crypto_flags=" -march=armv8-a+crypto" + # +crc: Apple Silicon enables __ARM_FEATURE_CRC32 by default, so milvus core/knowhere + # compile folly's F14 hash table in SimdAndCrc mode. 3rdparty deps (folly) must match, + # else folly::f14::F14LinkCheck is undefined at link time. + arm_crypto_flags=" -march=armv8-a+crypto+crc" fi export CFLAGS="${CFLAGS} -DTARGET_OS_OSX=1${arm_crypto_flags}" export CXXFLAGS="${CXXFLAGS} -DTARGET_OS_OSX=1${arm_crypto_flags}" diff --git a/scripts/setenv.sh b/scripts/setenv.sh index 874c262b44..5d367adf6c 100644 --- a/scripts/setenv.sh +++ b/scripts/setenv.sh @@ -60,8 +60,10 @@ case "${unameOut}" in export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$MILVUS_LIB_DIRS" export RPATH=$MILVUS_LIB_DIRS;; Darwin*) - # detect llvm version by valid list (supports LLVM 14-18) - for llvm_version in 18 17 16 15 14 NOT_FOUND ; do + # detect llvm version by valid list (supports LLVM 14-21) + # Prefer 19+ on macOS: libc++ 17/18 has a chrono operator<< that conflicts + # with arrow's vendored date library; libc++ 19 resolves it. + for llvm_version in 21 20 19 18 17 16 15 14 NOT_FOUND ; do if brew ls --versions llvm@${llvm_version} > /dev/null 2>&1; then break fi