From d69bdd288cdb2e219da8dc41992b5e3dc40e71f2 Mon Sep 17 00:00:00 2001 From: Xiaofan <83447078+xiaofan-luan@users.noreply.github.com> Date: Fri, 6 Mar 2026 23:19:21 +0800 Subject: [PATCH] fix: fix macOS 15 ARM64 compilation issues (#7437) (#47810) issue: #47809 --------- Signed-off-by: xiaofanluan Co-authored-by: Claude Opus 4.6 --- .../actions/macos-cache-restore/action.yaml | 12 +- .github/actions/macos-cache-save/action.yaml | 6 +- .github/workflows/code-checker.yaml | 10 +- .github/workflows/mac.yaml | 4 +- .github/workflows/publish-builder.yaml | 2 +- DEVELOPMENT.md | 10 +- README.md | 2 +- README_CN.md | 2 +- .../builder/cpu/amazonlinux2023/Dockerfile | 4 +- .../docker/builder/cpu/rockylinux8/Dockerfile | 39 ----- .../docker/builder/cpu/rockylinux9/Dockerfile | 55 +++++++ .../docker/builder/cpu/ubuntu20.04/Dockerfile | 13 +- .../docker/builder/cpu/ubuntu22.04/Dockerfile | 2 +- .../docker/builder/cpu/ubuntu24.04/Dockerfile | 56 ++++++++ build/docker/builder/entrypoint.sh | 4 + .../docker/builder/gpu/ubuntu20.04/Dockerfile | 2 +- .../docker/builder/gpu/ubuntu22.04/Dockerfile | 2 +- .../docker/milvus/amazonlinux2023/Dockerfile | 4 + .../{rockylinux8 => rockylinux9}/Dockerfile | 19 +-- build/docker/milvus/ubuntu20.04/Dockerfile | 4 + build/docker/milvus/ubuntu22.04/Dockerfile | 6 +- ci/jenkins/PR-for-go-sdk.groovy | 10 +- ci/jenkins/PR.groovy | 2 +- .../migration/backend/backup_header.pb.go | 2 +- internal/core/CMakeLists.txt | 16 +++ internal/core/conanfile.py | 62 ++++---- internal/core/lsan_suppressions.txt | 12 ++ internal/core/src/CMakeLists.txt | 11 -- internal/core/src/common/VectorArrayTest.cpp | 14 +- internal/core/src/common/logging_c.cpp | 2 +- internal/core/src/exec/expression/Expr.cpp | 2 +- .../exec/expression/GISFunctionFilterExpr.cpp | 6 +- .../core/src/exec/operator/RescoresNode.cpp | 3 +- internal/core/src/index/RTreeIndex.cpp | 14 +- internal/core/src/index/VectorDiskIndex.cpp | 8 +- internal/core/src/index/VectorMemIndex.cpp | 8 +- .../src/index/json_stats/JsonKeyStats.cpp | 2 +- internal/core/src/index/json_stats/utils.h | 5 +- internal/core/src/mmap/ChunkVector.h | 6 +- internal/core/src/mmap/ChunkVectorTest.cpp | 1 - internal/core/src/query/PlanProto.cpp | 2 +- internal/core/src/rescores/Utils.h | 2 +- .../src/segcore/SegmentGrowingIndexTest.cpp | 1 - .../segcore/SegmentGrowingStorageV2Test.cpp | 1 - .../core/src/segcore/SegmentGrowingTest.cpp | 1 - internal/core/src/storage/ChunkManager.cpp | 14 +- .../src/storage/minio/MinioChunkManager.cpp | 32 +++-- .../src/storage/minio/MinioChunkManager.h | 3 + .../core/thirdparty/knowhere/CMakeLists.txt | 48 +------ .../thirdparty/milvus-common/CMakeLists.txt | 3 +- .../thirdparty/milvus-storage/CMakeLists.txt | 97 ++++++------- internal/core/unittest/CMakeLists.txt | 11 +- internal/core/unittest/init_gtest.cpp | 21 +++ internal/core/unittest/test_query.cpp | 24 ++-- pkg/eventlog/event_log.pb.go | 2 +- pkg/eventlog/event_log_grpc.pb.go | 2 +- pkg/proto/cgopb/cgo_msg.pb.go | 2 +- pkg/proto/clusteringpb/clustering.pb.go | 2 +- pkg/proto/datapb/data_coord.pb.go | 2 +- pkg/proto/datapb/data_coord_grpc.pb.go | 2 +- pkg/proto/etcdpb/etcd_meta.pb.go | 2 +- pkg/proto/indexcgopb/index_cgo_msg.pb.go | 2 +- pkg/proto/indexpb/index_coord.pb.go | 2 +- pkg/proto/indexpb/index_coord_grpc.pb.go | 2 +- pkg/proto/internalpb/internal.pb.go | 2 +- pkg/proto/messagespb/messages.pb.go | 2 +- pkg/proto/modelservicepb/model_service.pb.go | 2 +- .../modelservicepb/model_service_grpc.pb.go | 2 +- pkg/proto/planpb/plan.pb.go | 2 +- pkg/proto/proxypb/proxy.pb.go | 2 +- pkg/proto/proxypb/proxy_grpc.pb.go | 2 +- pkg/proto/querypb/query_coord.pb.go | 2 +- pkg/proto/querypb/query_coord_grpc.pb.go | 2 +- pkg/proto/rootcoordpb/root_coord.pb.go | 2 +- pkg/proto/rootcoordpb/root_coord_grpc.pb.go | 2 +- pkg/proto/segcorepb/segcore.pb.go | 2 +- pkg/proto/streamingpb/streaming.pb.go | 2 +- pkg/proto/streamingpb/streaming_grpc.pb.go | 2 +- pkg/proto/workerpb/worker.pb.go | 2 +- pkg/proto/workerpb/worker_grpc.pb.go | 2 +- scripts/3rdparty_build.sh | 134 +++++++++++++++++- scripts/README.md | 2 +- scripts/check_proto_product.sh | 4 +- scripts/core_build.sh | 16 +-- scripts/install_deps.sh | 82 +++-------- scripts/install_deps_msys.sh | 2 +- scripts/run_cpp_codecov.sh | 10 ++ scripts/run_cpp_unittest.sh | 6 + scripts/run_cpp_ut.sh | 12 +- scripts/run_go_codecov.sh | 1 + scripts/run_intergration_test.sh | 3 +- scripts/setenv.sh | 48 +++++-- 92 files changed, 653 insertions(+), 423 deletions(-) delete mode 100644 build/docker/builder/cpu/rockylinux8/Dockerfile create mode 100644 build/docker/builder/cpu/rockylinux9/Dockerfile create mode 100644 build/docker/builder/cpu/ubuntu24.04/Dockerfile rename build/docker/milvus/{rockylinux8 => rockylinux9}/Dockerfile (67%) create mode 100644 internal/core/lsan_suppressions.txt diff --git a/.github/actions/macos-cache-restore/action.yaml b/.github/actions/macos-cache-restore/action.yaml index b081717ebd..5f57c089c5 100644 --- a/.github/actions/macos-cache-restore/action.yaml +++ b/.github/actions/macos-cache-restore/action.yaml @@ -20,8 +20,8 @@ runs: uses: actions/cache/restore@v4 with: path: /var/tmp/ccache - key: macos15-ccache-${{ env.corehash }} - restore-keys: macos15-ccache- + key: macos15-arm-ccache-${{ env.corehash }} + restore-keys: macos15-arm-ccache- - name: Mac Cache Go Mod Volumes id: go-mod-cache run: | @@ -35,8 +35,8 @@ runs: uses: actions/cache/restore@v4 with: path: ~/go/pkg/mod - key: macos15-go-mod-${{ steps.go-mod-cache.outputs.hash }} - restore-keys: macos15-go-mod- + key: macos15-arm-go-mod-${{ steps.go-mod-cache.outputs.hash }} + restore-keys: macos15-arm-go-mod- - name: Mac Cache Conan Packages id: conan-cache run: | @@ -50,5 +50,5 @@ runs: uses: actions/cache/restore@v4 with: path: ~/.conan - key: macos15-conan-${{ steps.conan-cache.outputs.hash }} - restore-keys: macos15-conan- + key: macos15-arm-conan-${{ steps.conan-cache.outputs.hash }} + restore-keys: macos15-arm-conan- diff --git a/.github/actions/macos-cache-save/action.yaml b/.github/actions/macos-cache-save/action.yaml index 98a54063be..87cf712b9c 100644 --- a/.github/actions/macos-cache-save/action.yaml +++ b/.github/actions/macos-cache-save/action.yaml @@ -29,7 +29,7 @@ runs: uses: actions/cache/save@v4 with: path: /var/tmp/ccache - key: macos15-ccache-${{ env.corehash }} + key: macos15-arm-ccache-${{ env.corehash }} - name: Generate Go Mod Hash id: go-mod-hash run: | @@ -43,7 +43,7 @@ runs: uses: actions/cache/save@v4 with: path: ~/go/pkg/mod - key: macos15-go-mod-${{ steps.go-mod-hash.outputs.hash }} + key: macos15-arm-go-mod-${{ steps.go-mod-hash.outputs.hash }} - name: Generate Conan Hash id: conan-hash run: | @@ -57,5 +57,5 @@ runs: uses: actions/cache/save@v4 with: path: ~/.conan - key: macos15-conan-${{ steps.conan-hash.outputs.hash }} + key: macos15-arm-conan-${{ steps.conan-hash.outputs.hash }} diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index 2a622366d6..de2a018298 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -101,8 +101,8 @@ jobs: rockylinux: if: ${{ false }} # skip for now - name: Code Checker rockylinux8 - # Run in amazonlinux docker + name: Code Checker rockylinux9 + # Run in rockylinux docker runs-on: ubuntu-latest timeout-minutes: 180 steps: @@ -120,13 +120,13 @@ jobs: - name: Download Caches uses: ./.github/actions/cache-restore with: - os: 'rockylinux8' + os: 'rockylinux9' - name: Code Check run: | - sed -i 's/ubuntu22.04/rockylinux8/g' .env + sed -i 's/ubuntu22.04/rockylinux9/g' .env ./build/builder.sh /bin/bash -c "make install" - name: Save Caches uses: ./.github/actions/cache-save if: github.event_name != 'pull_request' with: - os: 'rockylinux8' + os: 'rockylinux9' diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 3f4197de43..100716e95b 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -43,7 +43,7 @@ concurrency: jobs: mac: name: Code Checker MacOS - runs-on: macos-15-intel + runs-on: macos-15 timeout-minutes: 480 steps: - name: Checkout @@ -107,7 +107,7 @@ jobs: if [[ ! -d "${BREW_PREFIX}/opt/llvm" ]]; then ln -s ${BREW_PREFIX}/opt/llvm@17 ${BREW_PREFIX}/opt/llvm fi - pip3 install conan==1.64.1 + pip3 install conan==1.66.0 make milvus - name: Upload Cmake log uses: actions/upload-artifact@v4 diff --git a/.github/workflows/publish-builder.yaml b/.github/workflows/publish-builder.yaml index db5a6e9652..e9796b6f0e 100644 --- a/.github/workflows/publish-builder.yaml +++ b/.github/workflows/publish-builder.yaml @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu20.04, ubuntu22.04, amazonlinux2023, rockylinux8] + os: [ubuntu24.04, ubuntu22.04, amazonlinux2023, rockylinux9] env: OS_NAME: ${{ matrix.os }} IMAGE_ARCH: ${{ matrix.arch }} diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f3447d697b..ca2aa7a7fa 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -69,7 +69,7 @@ The following specification (either physical or virtual machine resources) is re |----------|----------|--------------|-------| | macOS | 12, 13, 14, 15 | Intel (x86_64), Apple Silicon (arm64) | Requires Homebrew | | Ubuntu | 20.04, 22.04, 24.04 | x86_64, arm64 | | -| Rocky Linux | 8, 9 | x86_64, arm64 | RHEL compatible | +| Rocky Linux | 9 | x86_64, arm64 | RHEL compatible | | Amazon Linux | 2023 | x86_64, arm64 | | | CentOS | 7 (legacy) | x86_64 | EOL, use Rocky Linux instead | @@ -79,8 +79,8 @@ The following specification (either physical or virtual machine resources) is re | Platform | Compiler | Supported Versions | |----------|----------|-------------------| -| macOS | LLVM/Clang | 14, 15, 16, 17, 18 | -| Linux | GCC | 9, 10, 11, 12, 13, 14 | +| macOS | LLVM/Clang | 15, 16, 17 | +| Linux | GCC | 11, 12, 13, 14 | **Tool Requirements:** @@ -172,7 +172,7 @@ Milvus uses Conan to manage third-party dependencies for c++. Install Conan: ```shell -pip install conan==1.64.1 +pip install conan==1.66.0 ``` Note: Conan version 2.x is not currently supported, please use version 1.64.1. @@ -480,7 +480,7 @@ A: This is a known issue on macOS 15+. Try using a newer version of LLVM: brew install llvm@17 ``` -The build scripts will automatically detect and use LLVM 14-18. +The build scripts will automatically detect and use LLVM 15-17. --- diff --git a/README.md b/README.md index 9299271105..4ebc05330c 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ Requirements: ```bash Go: >= 1.21 CMake: >= 3.26.4 && CMake < 4 - GCC: 9.5 + GCC: >= 11 Python: > 3.8 and <= 3.11 ``` diff --git a/README_CN.md b/README_CN.md index fded0d69de..50634d7ce5 100644 --- a/README_CN.md +++ b/README_CN.md @@ -72,7 +72,7 @@ Milvus 基于 [Apache 2.0 License](https://github.com/milvus-io/milvus/blob/mast ``` go: 1.21 cmake: >=3.18 -gcc: 7.5 +gcc: >= 11 protobuf: >=3.7 ``` diff --git a/build/docker/builder/cpu/amazonlinux2023/Dockerfile b/build/docker/builder/cpu/amazonlinux2023/Dockerfile index 940d3be2b1..92eb54d65c 100644 --- a/build/docker/builder/cpu/amazonlinux2023/Dockerfile +++ b/build/docker/builder/cpu/amazonlinux2023/Dockerfile @@ -15,7 +15,7 @@ ARG TARGETARCH RUN dnf install -y wget g++ gcc gdb libatomic libstdc++-static ninja-build git make zip unzip tar which \ autoconf automake python3 python3-pip perl-FindBin texinfo \ - pkg-config libuuid-devel libaio perl-IPC-Cmd libasan openblas-devel tzdata && \ + pkg-config libuuid-devel libaio perl-IPC-Cmd perl-Digest-SHA libasan openblas-devel tzdata && \ rm -rf /var/cache/yum/* ENV GOPATH /go @@ -27,7 +27,7 @@ RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARG go clean --modcache && \ chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) -RUN pip3 install conan==1.64.1 +RUN pip3 install conan==1.66.0 RUN echo "target arch $TARGETARCH" RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local diff --git a/build/docker/builder/cpu/rockylinux8/Dockerfile b/build/docker/builder/cpu/rockylinux8/Dockerfile deleted file mode 100644 index 63325db11b..0000000000 --- a/build/docker/builder/cpu/rockylinux8/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM rockylinux/rockylinux:8 as vcpkg-installer - -RUN dnf -y install curl wget tar zip unzip git \ - gcc gcc-c++ make cmake \ - perl-IPC-Cmd perl-Digest-SHA - -# install ninjia -RUN dnf -y update && \ - dnf -y install dnf-plugins-core && \ - dnf config-manager --set-enabled powertools && \ - dnf -y install ninja-build - -######################################################################################## -FROM rockylinux/rockylinux:8 - -ARG TARGETARCH - -RUN dnf install -y make cmake automake gcc gcc-c++ curl zip unzip tar git which \ - libaio libuuid-devel wget python3 python3-pip \ - pkg-config perl-IPC-Cmd perl-Digest-SHA libatomic libtool tzdata - -# install openblas-devel texinfo ninja -RUN dnf -y update && \ - dnf -y install dnf-plugins-core && \ - dnf config-manager --set-enabled powertools && \ - dnf -y install texinfo openblas-devel ninja-build - - -RUN pip3 install conan==1.64.1 -RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go -RUN curl https://sh.rustup.rs -sSf | \ - sh -s -- --default-toolchain=1.89 -y - -ENV PATH=/root/.cargo/bin:/usr/local/bin:/usr/local/go/bin:$PATH - -COPY --chown=0:0 build/docker/builder/entrypoint.sh / - -ENTRYPOINT [ "/entrypoint.sh" ] -CMD ["tail", "-f", "/dev/null"] diff --git a/build/docker/builder/cpu/rockylinux9/Dockerfile b/build/docker/builder/cpu/rockylinux9/Dockerfile new file mode 100644 index 0000000000..879671ff9b --- /dev/null +++ b/build/docker/builder/cpu/rockylinux9/Dockerfile @@ -0,0 +1,55 @@ +# Copyright (C) 2019-2022 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +FROM rockylinux/rockylinux:9 + +ARG TARGETARCH + +# Rocky 9 ships with GCC 11 and Python 3.9 by default. +# Enable CRB (CodeReady Builder) repo for openblas-devel, texinfo, etc. +RUN dnf install -y dnf-plugins-core epel-release && \ + dnf config-manager --set-enabled crb && \ + dnf install -y \ + wget g++ gcc gdb libatomic libstdc++-static ninja-build git make zip unzip tar which \ + autoconf automake python3 python3-pip perl-FindBin texinfo ccache \ + pkg-config libuuid-devel libaio libaio-devel perl-IPC-Cmd perl-Digest-SHA \ + libasan openblas-devel tzdata libtool && \ + rm -rf /var/cache/dnf/* && \ + for f in /usr/include/openblas/*.h; do ln -sf "$f" /usr/include/$(basename "$f"); done + +ENV GOPATH /go +ENV GOROOT /usr/local/go +ENV GO111MODULE on +ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ + mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ + go clean --modcache && \ + chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) + +RUN pip3 install conan==1.66.0 + +RUN echo "target arch $TARGETARCH" +RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local + +# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild +RUN mkdir -p /home/milvus/.vscode-server/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ + && chmod -R 777 /home/milvus + +COPY --chown=0:0 build/docker/builder/entrypoint.sh / + +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- --default-toolchain=1.89 -y + +ENV PATH=/root/.cargo/bin:$PATH + +ENTRYPOINT [ "/entrypoint.sh" ] +CMD ["tail", "-f", "/dev/null"] diff --git a/build/docker/builder/cpu/ubuntu20.04/Dockerfile b/build/docker/builder/cpu/ubuntu20.04/Dockerfile index 60a280d68f..cae473d860 100644 --- a/build/docker/builder/cpu/ubuntu20.04/Dockerfile +++ b/build/docker/builder/cpu/ubuntu20.04/Dockerfile @@ -16,14 +16,23 @@ ARG TARGETARCH RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 \ g++ gcc gdb gdbserver ninja-build git make ccache libssl-dev zlib1g-dev zip unzip \ clang-format-12 clang-tidy-12 lcov libtool m4 autoconf automake python3 python3-pip \ - pkg-config uuid-dev libaio-dev libopenblas-dev && \ + pkg-config uuid-dev libaio-dev libopenblas-dev software-properties-common && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \ ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ echo "Etc/UTC" > /etc/timezone && \ apt-get remove --purge -y && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install conan==1.64.1 +# Upgrade GCC to 11 (focal default is 9, which folly 2024.08.12.00 rejects) +RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ + apt-get update && apt-get install -y gcc-11 g++-11 && \ + cd /usr/bin && \ + unlink gcc && ln -s gcc-11 gcc && \ + unlink g++ && ln -s g++-11 g++ && \ + unlink gcov && ln -s gcov-11 gcov && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install conan==1.66.0 RUN echo "target arch $TARGETARCH" RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local diff --git a/build/docker/builder/cpu/ubuntu22.04/Dockerfile b/build/docker/builder/cpu/ubuntu22.04/Dockerfile index 0dc37f7f40..ecb8b8cdd0 100644 --- a/build/docker/builder/cpu/ubuntu22.04/Dockerfile +++ b/build/docker/builder/cpu/ubuntu22.04/Dockerfile @@ -29,7 +29,7 @@ RUN apt-get update && apt-get install -y gcc-12 g++-12 && cd /usr/bin \ && unlink g++ && ln -s g++-12 g++ \ && unlink gcov && ln -s gcov-12 gcov -RUN pip3 install conan==1.64.1 +RUN pip3 install conan==1.66.0 RUN echo "target arch $TARGETARCH" RUN wget -qO- "https://cmake.org/files/v3.31/cmake-3.31.8-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local diff --git a/build/docker/builder/cpu/ubuntu24.04/Dockerfile b/build/docker/builder/cpu/ubuntu24.04/Dockerfile new file mode 100644 index 0000000000..689f3d3108 --- /dev/null +++ b/build/docker/builder/cpu/ubuntu24.04/Dockerfile @@ -0,0 +1,56 @@ +# Copyright (C) 2019-2022 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +FROM ubuntu:noble-20240530 + +ARG TARGETARCH + +RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 \ + g++ gcc gdb gdbserver ninja-build git make ccache libssl-dev zlib1g-dev zip unzip \ + clang-format-18 clang-tidy-18 lcov libtool m4 autoconf automake python3 python3-pip pipx \ + pkg-config uuid-dev libaio-dev libopenblas-dev && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \ + ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + echo "Etc/UTC" > /etc/timezone && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +# Ubuntu 24.04 ships GCC 13 by default, no upgrade needed + +RUN pip3 install --break-system-packages conan==1.66.0 + +RUN echo "target arch $TARGETARCH" +RUN wget -qO- "https://cmake.org/files/v3.31/cmake-3.31.8-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local + +# Install Go +ENV GOPATH /go +ENV GOROOT /usr/local/go +ENV GO111MODULE on +ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ + mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ + go clean --modcache && \ + chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) + +# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild +RUN mkdir -p /home/milvus/.vscode-server/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ + && chmod -R 777 /home/milvus + +COPY --chown=0:0 build/docker/builder/entrypoint.sh / + +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- --default-toolchain=1.89 -y + +ENV PATH=/root/.cargo/bin:$PATH + +ENTRYPOINT [ "/entrypoint.sh" ] +CMD ["tail", "-f", "/dev/null"] diff --git a/build/docker/builder/entrypoint.sh b/build/docker/builder/entrypoint.sh index 67f25c3f53..1bc0e762b4 100755 --- a/build/docker/builder/entrypoint.sh +++ b/build/docker/builder/entrypoint.sh @@ -26,6 +26,10 @@ if [ -f "/etc/profile.d/devtoolset-11.sh" ]; then source "/etc/profile.d/devtoolset-11.sh" fi +if [ -f "/opt/rh/gcc-toolset-11/enable" ]; then + source "/opt/rh/gcc-toolset-11/enable" +fi + if [ -f "/etc/profile.d/llvm-toolset-11.sh" ]; then source "/etc/profile.d/llvm-toolset-11.sh" fi diff --git a/build/docker/builder/gpu/ubuntu20.04/Dockerfile b/build/docker/builder/gpu/ubuntu20.04/Dockerfile index 0ba4bb02e0..602dd83f12 100644 --- a/build/docker/builder/gpu/ubuntu20.04/Dockerfile +++ b/build/docker/builder/gpu/ubuntu20.04/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce apt-get remove --purge -y && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install conan==1.64.1 +RUN pip3 install conan==1.66.0 # Install openblas # RUN wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \ diff --git a/build/docker/builder/gpu/ubuntu22.04/Dockerfile b/build/docker/builder/gpu/ubuntu22.04/Dockerfile index 56abccacb8..adc1018b71 100644 --- a/build/docker/builder/gpu/ubuntu22.04/Dockerfile +++ b/build/docker/builder/gpu/ubuntu22.04/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce # Install go RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go # Install conan -RUN pip3 install conan==1.64.1 +RUN pip3 install conan==1.66.0 # Install rust RUN curl https://sh.rustup.rs -sSf | \ sh -s -- --default-toolchain=1.89 -y diff --git a/build/docker/milvus/amazonlinux2023/Dockerfile b/build/docker/milvus/amazonlinux2023/Dockerfile index 1f2c96f980..203e8d1982 100644 --- a/build/docker/milvus/amazonlinux2023/Dockerfile +++ b/build/docker/milvus/amazonlinux2023/Dockerfile @@ -35,9 +35,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so ENV MALLOC_CONF=background_thread:true # Change user to milvus +# protobuf 5.x RepeatedField adds container annotations that trigger ASAN +# container-overflow false positives during static initialization, disable it. +ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0 RUN groupadd -r milvus && useradd -r -g milvus milvus && \ mkdir -p /var/lib/milvus && \ chown -R milvus:milvus /milvus /var/lib/milvus +ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0 USER milvus:milvus ENTRYPOINT ["/tini", "--"] diff --git a/build/docker/milvus/rockylinux8/Dockerfile b/build/docker/milvus/rockylinux9/Dockerfile similarity index 67% rename from build/docker/milvus/rockylinux8/Dockerfile rename to build/docker/milvus/rockylinux9/Dockerfile index d8c073d651..0f223b709a 100644 --- a/build/docker/milvus/rockylinux8/Dockerfile +++ b/build/docker/milvus/rockylinux9/Dockerfile @@ -1,5 +1,4 @@ # Copyright (C) 2019-2020 Zilliz. All rights reserved. - # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at @@ -10,17 +9,17 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License. -FROM rockylinux/rockylinux:8 +FROM rockylinux/rockylinux:9 ARG TARGETARCH ARG MILVUS_ASAN_LIB -RUN dnf install -y wget libgomp libaio libatomic tzdata - -# install openblas-devel -RUN dnf -y install dnf-plugins-core && \ - dnf config-manager --set-enabled powertools && \ - dnf -y install openblas-devel +# Enable CRB repo for openblas-devel +RUN dnf install -y dnf-plugins-core && \ + dnf config-manager --set-enabled crb && \ + dnf install -y wget libgomp libaio libatomic openblas-devel tzdata && \ + rm -rf /var/cache/dnf/* && \ + for f in /usr/include/openblas/*.h; do ln -sf "$f" /usr/include/$(basename "$f"); done # Add Tini RUN curl -L -o /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini-$TARGETARCH && \ @@ -39,9 +38,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so ENV MALLOC_CONF=background_thread:true # Change user to milvus +# protobuf 5.x RepeatedField adds container annotations that trigger ASAN +# container-overflow false positives during static initialization, disable it. +ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0 RUN groupadd -r milvus && useradd -r -g milvus milvus && \ mkdir -p /var/lib/milvus && \ chown -R milvus:milvus /milvus /var/lib/milvus +ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0 USER milvus:milvus ENTRYPOINT ["/tini", "--"] diff --git a/build/docker/milvus/ubuntu20.04/Dockerfile b/build/docker/milvus/ubuntu20.04/Dockerfile index 088e9ee343..9ba0a9612a 100644 --- a/build/docker/milvus/ubuntu20.04/Dockerfile +++ b/build/docker/milvus/ubuntu20.04/Dockerfile @@ -43,9 +43,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so ENV MALLOC_CONF=background_thread:true # Change user to milvus +# protobuf 5.x RepeatedField adds container annotations that trigger ASAN +# container-overflow false positives during static initialization, disable it. +ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0 RUN groupadd -r milvus && useradd -r -g milvus milvus && \ mkdir -p /var/lib/milvus && \ chown -R milvus:milvus /milvus /var/lib/milvus +ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0 USER milvus:milvus ENTRYPOINT ["/tini", "--"] diff --git a/build/docker/milvus/ubuntu22.04/Dockerfile b/build/docker/milvus/ubuntu22.04/Dockerfile index b7816fed20..2b57fd7d3a 100644 --- a/build/docker/milvus/ubuntu22.04/Dockerfile +++ b/build/docker/milvus/ubuntu22.04/Dockerfile @@ -43,11 +43,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so ENV MALLOC_CONF=background_thread:true # Change user to milvus -ENV ASAN_OPTIONS=detect_leaks=0 +# protobuf 5.x RepeatedField adds container annotations that trigger ASAN +# container-overflow false positives during static initialization, disable it. +ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0 RUN groupadd -r milvus && useradd -r -g milvus milvus && \ mkdir -p /var/lib/milvus && \ chown -R milvus:milvus /milvus /var/lib/milvus -ENV ASAN_OPTIONS=detect_leaks=1 +ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0 USER milvus:milvus ENTRYPOINT ["/tini", "--"] diff --git a/ci/jenkins/PR-for-go-sdk.groovy b/ci/jenkins/PR-for-go-sdk.groovy index 5cc82c7c9e..d7a0d4b7d8 100644 --- a/ci/jenkins/PR-for-go-sdk.groovy +++ b/ci/jenkins/PR-for-go-sdk.groovy @@ -11,7 +11,7 @@ pipeline { buildDiscarder logRotator(artifactDaysToKeepStr: '30') preserveStashes(buildCount: 5) disableConcurrentBuilds(abortPrevious: true) - timeout(time: 2, unit: 'HOURS') + timeout(time: 4, unit: 'HOURS') throttleJobProperty( categories: ['go-sdk'], throttleEnabled: true, @@ -58,7 +58,9 @@ pipeline { pullRequestNumber: "$env.CHANGE_ID", suppress_suffix_of_image_tag: true, make_cmd: "make clean && make jobs=8 install use_disk_index=ON", - images: '["milvus","gotestsum","helm"]' + images: '["milvus","gotestsum","helm"]', + tekton_log_timeout: '30m', + tekton_pipeline_timeout: '3h' milvus_image_tag = tekton.query_result job_name, 'milvus-image-tag' milvus_sdk_go_image = tekton.query_result job_name, 'gotestsum-image-fqdn' @@ -104,7 +106,9 @@ pipeline { milvus_sdk_go_image: milvus_sdk_go_image, helm_image: helm_image, milvus_deployment_option: milvus_deployment_option, - verbose: 'false' + verbose: 'false', + tekton_log_timeout: '30m', + tekton_pipeline_timeout: '3h' } } } diff --git a/ci/jenkins/PR.groovy b/ci/jenkins/PR.groovy index 121b5b1d7d..352e414d1b 100644 --- a/ci/jenkins/PR.groovy +++ b/ci/jenkins/PR.groovy @@ -10,7 +10,7 @@ pipeline { buildDiscarder logRotator(artifactDaysToKeepStr: '30') preserveStashes(buildCount: 5) disableConcurrentBuilds(abortPrevious: true) - timeout(time: 3, unit: 'HOURS') + timeout(time: 4, unit: 'HOURS') throttleJobProperty( categories: ['cpu-e2e'], throttleEnabled: true, diff --git a/cmd/tools/migration/backend/backup_header.pb.go b/cmd/tools/migration/backend/backup_header.pb.go index 83d9a37541..b67a235870 100644 --- a/cmd/tools/migration/backend/backup_header.pb.go +++ b/cmd/tools/migration/backend/backup_header.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: backup_header.proto package backend diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index f4e9987a97..9a2b087fd3 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -97,6 +97,11 @@ set( MILVUS_THIRDPARTY_SRC ${PROJECT_SOURCE_DIR}/thirdparty ) # self-installed dynamic libraries will be correctly linked by excutable set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" "${CMAKE_INSTALL_PREFIX}/lib64" ) set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE ) +# FetchContent-built shared libs (libbson, libbsoncxx, etc.) are placed in +# ${CMAKE_BINARY_DIR}/lib. The linker needs -rpath-link to resolve transitive +# shared library dependencies (e.g. libbsoncxx.so -> libbson-1.0.so) when +# linking test binaries. +set( CMAKE_BUILD_RPATH "${CMAKE_BINARY_DIR}/lib" ) # **************************** Dependencies **************************** list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/conan ) @@ -164,6 +169,17 @@ if ( APPLE ) # Disable fixup chains to avoid __DATA_CONST segment issues with third-party libraries set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no_fixup_chains") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-no_fixup_chains") + + # Folly's Exception.cpp references __cxa_increment/decrement_exception_refcount + # from libc++abi, which lives in /lib/c++/ and is not on the default + # linker search path. Derive the path from CMAKE_CXX_COMPILER. + get_filename_component(_CXX_BIN_DIR "${CMAKE_CXX_COMPILER}" DIRECTORY) + get_filename_component(_LLVM_ROOT "${_CXX_BIN_DIR}" DIRECTORY) + set(_LLVM_CXXABI_DIR "${_LLVM_ROOT}/lib/c++") + if(EXISTS "${_LLVM_CXXABI_DIR}/libc++abi.dylib") + link_directories("${_LLVM_CXXABI_DIR}") + link_libraries(c++abi) + endif() endif () # **************************** Coding style check tools **************************** diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py index 65de239851..103e98c86d 100644 --- a/internal/core/conanfile.py +++ b/internal/core/conanfile.py @@ -11,44 +11,47 @@ class MilvusConan(ConanFile): "nlohmann_json/3.11.3#ffb9e9236619f1c883e36662f944345d", "zstd/1.5.5#34e9debe03bf0964834a09dfbc31a5dd", "lz4/1.9.4#c5afb86edd69ac0df30e3a9e192e43db", - "snappy/1.1.9#0519333fef284acd04806243de7d3070", + "snappy/1.2.1#dfb7f7e837525210762d74e96eb4a3fc", "arrow/17.0.0@milvus/dev-2.6#7af258a853e20887f9969f713110aac8", - "openssl/3.1.2#02594c4c0a6e2b4feb3cd15119993597", + "openssl/3.3.2#9f9f130d58e7c13e76bb8a559f0a6a8b", "googleapis/cci.20221108#65604e1b3b9a6b363044da625b201a2a", "gtest/1.13.0#f9548be18a41ccc6367efcb8146e92be", "benchmark/1.7.0#459f3bb1a64400a886ba43047576df3c", - "protobuf/3.21.4#fd372371d994b8585742ca42c12337f9", + "protobuf/5.27.0@milvus/dev#6fff8583e2fe32babef04a9097f1d581", "yaml-cpp/0.7.0#9c87b3998de893cf2e5a08ad09a7a6e0", "marisa/0.2.6#68446854f5a420672d21f21191f8e5af", - "zlib/1.2.13#df233e6bed99052f285331b9f54d9070", - "libcurl/7.86.0#bbc887fae3341b3cb776c601f814df05", - "glog/0.6.0#d22ebf9111fed68de86b0fa6bf6f9c3f", - "fmt/9.1.0#95259249fb7ef8c6b5674a40b00abba3", + "zlib/1.3.1#f52e03ae3d251dec704634230cd806a2", + "libcurl/8.10.1#a3113369c86086b0e84231844e7ed0a9", + "libevent/2.1.12#b6333a128075d75a3614bd8418bf2099", + "glog/0.7.1#83889ae482004d816ecac59dfef0d65a", + "fmt/11.0.2#5c7438ef4d5d69ab106a41e460ce11f3", "gflags/2.2.2#b15c28c567c7ade7449cf994168a559f", - "double-conversion/3.2.1#640e35791a4bac95b0545e2f54b7aceb", + "double-conversion/3.3.0#33321c201741cc32b51169c6d2d05e60", "libsodium/cci.20220430#7429a9e5351cc67bea3537229921714d", "xsimd/9.0.1#ac9fd02a381698c4e08c5c4ca03b73e1", - "xz_utils/5.4.0#a6d90890193dc851fa0d470163271c7a", - "prometheus-cpp/1.1.0#ea9b101cb785943adb40ad82eda7856c", + "xz_utils/5.4.5#b885d1d79c9d30cff3803f7f551dbe66", + "prometheus-cpp/1.2.4#0918d66c13f97acb7809759f9de49b3f", "re2/20230301#f8efaf45f98d0193cd0b2ea08b6b4060", - "folly/2023.10.30.08@milvus/dev#81d7729cd4013a1b708af3340a3b04d9", - "google-cloud-cpp/2.28.0#cf2bffe2264488b6c1153fae2a8db095", - "opentelemetry-cpp/1.8.1.1@milvus/2.4#7345034855d593047826b0c74d9a0ced", + "folly/2024.08.12.00@milvus/dev#e09fc71826ce6b4568441910665f0889", + "google-cloud-cpp/2.28.0@milvus/dev#25e69d743269d6c9ae5bf676af2174dc", + "opentelemetry-cpp/1.23.0@milvus/dev#bcd65b63b8db8447178ed93bbc94dcc0", "librdkafka/1.9.1#e24dcbb0a1684dcf5a56d8d0692ceef3", - "abseil/20230125.3#dad7cc4c83bbd44c1f1cc9cc4d97ac88", + "abseil/20250127.0#e6c46096070cc3fd060e95a837cd3fb2", "roaring/3.0.0#25a703f80eda0764a31ef939229e202d", - "grpc/1.50.1@milvus/dev#75103960d1cac300cf425ccfccceac08", + "grpc/1.67.1@milvus/dev#5aa62c51bced448b83d7db9e5b3a13c7", + "rapidjson/cci.20230929#624c0094d741e6a3749d2e44d834b96c", "simde/0.8.2#5e1edfd5cba92f25d79bf6ef4616b972", "xxhash/0.8.3#199e63ab9800302c232d030b27accec0", "unordered_dense/4.4.0#6a855c992618cc4c63019109a2e47298", "geos/3.12.0#0b177c90c25a8ca210578fb9e2899c37", "icu/74.2#cd1937b9561b8950a2ae6311284c5813", - "libavrocpp/1.12.1@milvus/dev", + "libavrocpp/1.12.1.1@milvus/dev#a77043b1b435c3abef7b45710d05b300", ) generators = ("cmake", "cmake_find_package") default_options = { - "openssl:shared": True, + "openssl:shared": False, + "openssl:no_apps": True, "libevent:shared": True, "double-conversion:shared": True, "folly:shared": True, @@ -71,6 +74,7 @@ class MilvusConan(ConanFile): "arrow:with_azure": True, "arrow:with_s3": True, "arrow:encryption": True, + "protobuf:shared": True, "aws-sdk-cpp:config": True, "aws-sdk-cpp:text-to-speech": False, "aws-sdk-cpp:transfer": False, @@ -81,39 +85,31 @@ class MilvusConan(ConanFile): "glog:with_gflags": True, "glog:shared": True, "prometheus-cpp:with_pull": False, - "fmt:header_only": True, + "fmt:header_only": False, "onetbb:tbbmalloc": False, "onetbb:tbbproxy": False, - "gdal:shared": True, + "gflags:shared": True, + "gdal:shared": False, "gdal:fPIC": True, "icu:shared": False, "icu:data_packaging": "library", + "xz_utils:shared": True, + "opentelemetry-cpp:with_stl": True, } def configure(self): if self.settings.arch not in ("x86_64", "x86"): del self.options["folly"].use_sse4_2 if self.settings.os == "Macos": - # By default abseil use static link but can not be compatible with macos X86 - self.options["abseil"].shared = True + # abseil static linking on macOS (previously shared for X86 compat) self.options["arrow"].with_jemalloc = False - # Disable Arrow's S3 support on macOS - the Conan aws-c-* packages - # (aws-c-cal, aws-c-io) use deprecated macOS Security framework APIs - # that were removed in macOS 15+. Instead, milvus-storage provides - # S3 support using Homebrew's aws-sdk-cpp (1.11.735+) which has - # the compatibility fixes. - self.options["arrow"].with_s3 = False # Use OpenSSL for libcurl on macOS self.options["libcurl"].with_ssl = "openssl" def requirements(self): if self.settings.os != "Macos": - self.requires("libunwind/1.7.2") - # On Linux, use Conan's aws-sdk-cpp (Arrow and milvus-storage both use it) - self.requires("aws-sdk-cpp/1.11.352@milvus/dev") - # On macOS, S3 support is provided by milvus-storage using Homebrew's - # aws-sdk-cpp (brew install aws-sdk-cpp). Arrow's S3 is disabled to avoid - # pulling in incompatible aws-c-* packages from Conan. + self.requires("libunwind/1.8.1#97965ef7da98cf1662e14219b14134f7") + self.requires("aws-sdk-cpp/1.11.692@milvus/dev#1e17deac19383217d291a01c23147b33") def imports(self): self.copy("*.dylib", "../lib", "lib") diff --git a/internal/core/lsan_suppressions.txt b/internal/core/lsan_suppressions.txt new file mode 100644 index 0000000000..5004f43395 --- /dev/null +++ b/internal/core/lsan_suppressions.txt @@ -0,0 +1,12 @@ +# LeakSanitizer suppressions for Milvus C++ unit tests +# +# These suppress known false positives caused by C++ runtime exception +# object caching in uninstrumented shared libraries (protobuf, abseil, +# grpc, etc.). The exception objects are properly caught and handled +# during execution, but their thread-local caches appear as leaks +# at program exit because LeakSanitizer cannot track allocations +# across shared library boundaries without frame pointers. +# +# See: https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer#suppressions + +leak:__cxa_allocate_exception diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index 6acc371155..3ecae4fc42 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -38,17 +38,6 @@ include_directories( ${MILVUS_STORAGE_INCLUDE_DIR} ) -if(APPLE) - # On macOS, aws-sdk-cpp comes from Homebrew instead of Conan. - # Add Homebrew include path AFTER Conan includes so Conan headers take precedence, - # avoiding conflicts (e.g., Boost version mismatch). - if(EXISTS "/opt/homebrew/include/aws") - include_directories("/opt/homebrew/include") - elseif(EXISTS "/usr/local/include/aws") - include_directories("/usr/local/include") - endif() -endif() - add_subdirectory( pb ) add_subdirectory( config ) add_subdirectory( common ) diff --git a/internal/core/src/common/VectorArrayTest.cpp b/internal/core/src/common/VectorArrayTest.cpp index a63a977c75..ef6657c56c 100644 --- a/internal/core/src/common/VectorArrayTest.cpp +++ b/internal/core/src/common/VectorArrayTest.cpp @@ -29,8 +29,8 @@ using namespace milvus; TEST(VectorArray, TestSchema) { - namespace pb = milvus::proto; - pb::schema::CollectionSchema proto; + namespace milvus_pb = milvus::proto; + milvus_pb::schema::CollectionSchema proto; proto.set_name("col"); proto.set_description("asdfhsalkgfhsadg"); auto dim = 16; @@ -48,7 +48,7 @@ TEST(VectorArray, TestSchema) { field->set_fieldid(100); field->set_is_primary_key(true); field->set_description("asdgfsagf"); - field->set_data_type(pb::schema::DataType::Int64); + field->set_data_type(milvus_pb::schema::DataType::Int64); } { @@ -60,14 +60,14 @@ TEST(VectorArray, TestSchema) { field->set_name("struct_key"); field->set_nullable(false); field->set_fieldid(102); - field->set_data_type(pb::schema::DataType::Array); - field->set_element_type(pb::schema::DataType::Int64); + field->set_data_type(milvus_pb::schema::DataType::Array); + field->set_element_type(milvus_pb::schema::DataType::Int64); auto field2 = struct_field->add_fields(); field2->set_name("struct_float_vec"); field2->set_fieldid(103); - field2->set_data_type(pb::schema::DataType::ArrayOfVector); - field2->set_element_type(pb::schema::DataType::FloatVector); + field2->set_data_type(milvus_pb::schema::DataType::ArrayOfVector); + field2->set_element_type(milvus_pb::schema::DataType::FloatVector); auto param = field2->add_type_params(); param->set_key("dim"); param->set_value("16"); diff --git a/internal/core/src/common/logging_c.cpp b/internal/core/src/common/logging_c.cpp index 7aa161c0ba..63ad8355c4 100644 --- a/internal/core/src/common/logging_c.cpp +++ b/internal/core/src/common/logging_c.cpp @@ -125,6 +125,6 @@ InitGoogleLoggingWithoutZapSink() { void GoogleLoggingAtLevel(int severity, const char* msg) { - google::LogAtLevel(severity, msg); + google::LogAtLevel(static_cast(severity), msg); } } \ No newline at end of file diff --git a/internal/core/src/exec/expression/Expr.cpp b/internal/core/src/exec/expression/Expr.cpp index e9f6cca751..d6626e42ef 100644 --- a/internal/core/src/exec/expression/Expr.cpp +++ b/internal/core/src/exec/expression/Expr.cpp @@ -98,7 +98,7 @@ ExprSet::Eval(int32_t begin, // Create TTL field filtering expression if schema has TTL field configured // Returns a single OR expression: ttl_field is null OR ttl_field > physical_us // This means: keep entities with null TTL (never expire) OR entities with TTL > current time (not expired) -inline expr::TypedExprPtr +expr::TypedExprPtr CreateTTLFieldFilterExpression(QueryContext* query_context) { auto segment = query_context->get_segment(); auto& schema = segment->get_schema(); diff --git a/internal/core/src/exec/expression/GISFunctionFilterExpr.cpp b/internal/core/src/exec/expression/GISFunctionFilterExpr.cpp index c0f2e8e1c9..992c2e88bd 100644 --- a/internal/core/src/exec/expression/GISFunctionFilterExpr.cpp +++ b/internal/core/src/exec/expression/GISFunctionFilterExpr.cpp @@ -345,7 +345,7 @@ PhyGISFunctionFilterExpr::EvalForDataSegment() { default: { ThrowInfo(NotImplemented, "internal error: unknown GIS op : {}", - expr_->op_); + static_cast(expr_->op_)); } } return res_vec; @@ -455,7 +455,9 @@ PhyGISFunctionFilterExpr::EvalForIndexSegment() { // Distance-based operation - no prepared version return left.dwithin(query_geometry, expr_->distance_); default: - ThrowInfo(NotImplemented, "unknown GIS op : {}", expr_->op_); + ThrowInfo(NotImplemented, + "unknown GIS op : {}", + static_cast(expr_->op_)); } }; diff --git a/internal/core/src/exec/operator/RescoresNode.cpp b/internal/core/src/exec/operator/RescoresNode.cpp index 12076a2a91..ec3b86be9c 100644 --- a/internal/core/src/exec/operator/RescoresNode.cpp +++ b/internal/core/src/exec/operator/RescoresNode.cpp @@ -218,7 +218,8 @@ PhyRescoresNode::GetOutput() { break; default: ThrowInfo(ErrorCode::UnexpectedError, - fmt::format("unknown boost boost mode: {}", boost_mode)); + fmt::format("unknown boost boost mode: {}", + static_cast(boost_mode))); } knowhere::MetricType metric_type = query_context_->get_metric_type(); diff --git a/internal/core/src/index/RTreeIndex.cpp b/internal/core/src/index/RTreeIndex.cpp index ecd1ab60d2..135bacda53 100644 --- a/internal/core/src/index/RTreeIndex.cpp +++ b/internal/core/src/index/RTreeIndex.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include "boost/filesystem/directory.hpp" @@ -139,7 +141,7 @@ RTreeIndex::Load(milvus::tracer::TraceContext ctx, const Config& config) { }; auto fill_null_offsets = [&](const uint8_t* data, int64_t size) { - folly::SharedMutexWritePriority::WriteHolder lock(mutex_); + std::unique_lock lock(mutex_); null_offset_.resize((size_t)size / sizeof(size_t)); memcpy(null_offset_.data(), data, (size_t)size); }; @@ -295,7 +297,7 @@ RTreeIndex::BuildWithFieldData( total_rows += n; } if (!local_nulls.empty()) { - folly::SharedMutexWritePriority::WriteHolder lock(mutex_); + std::unique_lock lock(mutex_); null_offset_.reserve(null_offset_.size() + local_nulls.size()); null_offset_.insert( null_offset_.end(), local_nulls.begin(), local_nulls.end()); @@ -373,7 +375,7 @@ RTreeIndex::Upload(const Config& config) { template BinarySet RTreeIndex::Serialize(const Config& config) { - folly::SharedMutexWritePriority::ReadHolder lock(mutex_); + std::shared_lock lock(mutex_); auto bytes = null_offset_.size() * sizeof(size_t); BinarySet res_set; if (bytes > 0) { @@ -412,7 +414,7 @@ const TargetBitmap RTreeIndex::IsNull() { int64_t count = Count(); TargetBitmap bitset(count); - folly::SharedMutexWritePriority::ReadHolder lock(mutex_); + std::shared_lock lock(mutex_); auto end = std::lower_bound( null_offset_.begin(), null_offset_.end(), static_cast(count)); for (auto it = null_offset_.begin(); it != end; ++it) { @@ -426,7 +428,7 @@ TargetBitmap RTreeIndex::IsNotNull() { int64_t count = Count(); TargetBitmap bitset(count, true); - folly::SharedMutexWritePriority::ReadHolder lock(mutex_); + std::shared_lock lock(mutex_); auto end = std::lower_bound( null_offset_.begin(), null_offset_.end(), static_cast(count)); for (auto it = null_offset_.begin(); it != end; ++it) { @@ -603,7 +605,7 @@ RTreeIndex::AddGeometry(const std::string& wkb_data, int64_t row_offset) { LOG_DEBUG("Added geometry at row offset {}", row_offset); } else { // Handle null geometry - folly::SharedMutexWritePriority::WriteHolder lock(mutex_); + std::unique_lock lock(mutex_); null_offset_.push_back(static_cast(row_offset)); // Update total row count diff --git a/internal/core/src/index/VectorDiskIndex.cpp b/internal/core/src/index/VectorDiskIndex.cpp index e2705386ab..0de13e20ae 100644 --- a/internal/core/src/index/VectorDiskIndex.cpp +++ b/internal/core/src/index/VectorDiskIndex.cpp @@ -119,8 +119,10 @@ VectorDiskAnnIndex::Load(milvus::tracer::TraceContext ctx, { auto read_file_span = milvus::tracer::StartSpan("SegCoreReadDiskIndexFile", &ctx); + opentelemetry::nostd::shared_ptr + read_file_nostd_span(read_file_span); auto read_scope = - milvus::tracer::GetTracer()->WithActiveSpan(read_file_span); + opentelemetry::trace::Tracer::WithActiveSpan(read_file_nostd_span); auto index_files = GetValueFromConfig>(config, "index_files"); AssertInfo(index_files.has_value(), @@ -139,8 +141,10 @@ VectorDiskAnnIndex::Load(milvus::tracer::TraceContext ctx, // start engine load index span auto span_load_engine = milvus::tracer::StartSpan("SegCoreEngineLoadDiskIndex", &ctx); + opentelemetry::nostd::shared_ptr + nostd_span_load_engine(span_load_engine); auto engine_scope = - milvus::tracer::GetTracer()->WithActiveSpan(span_load_engine); + opentelemetry::trace::Tracer::WithActiveSpan(nostd_span_load_engine); auto stat = index_.Deserialize(knowhere::BinarySet(), load_config); if (stat != knowhere::Status::success) ThrowInfo(ErrorCode::UnexpectedError, diff --git a/internal/core/src/index/VectorMemIndex.cpp b/internal/core/src/index/VectorMemIndex.cpp index 6b634ef921..6d7274124d 100644 --- a/internal/core/src/index/VectorMemIndex.cpp +++ b/internal/core/src/index/VectorMemIndex.cpp @@ -264,8 +264,10 @@ VectorMemIndex::Load(milvus::tracer::TraceContext ctx, { auto read_file_span = milvus::tracer::StartSpan("SegCoreReadIndexFile", &ctx); + opentelemetry::nostd::shared_ptr + read_file_nostd_span(read_file_span); auto read_scope = - milvus::tracer::GetTracer()->WithActiveSpan(read_file_span); + opentelemetry::trace::Tracer::WithActiveSpan(read_file_nostd_span); LOG_INFO("load with slice meta: {}", !slice_meta_filepath.empty()); auto load_priority = @@ -348,8 +350,10 @@ VectorMemIndex::Load(milvus::tracer::TraceContext ctx, // start engine load index span auto span_load_engine = milvus::tracer::StartSpan("SegCoreEngineLoadIndex", &ctx); + opentelemetry::nostd::shared_ptr + nostd_span_load_engine(span_load_engine); auto engine_scope = - milvus::tracer::GetTracer()->WithActiveSpan(span_load_engine); + opentelemetry::trace::Tracer::WithActiveSpan(nostd_span_load_engine); LOG_INFO("load index into Knowhere..."); LoadWithoutAssemble(binary_set, config); span_load_engine->End(); diff --git a/internal/core/src/index/json_stats/JsonKeyStats.cpp b/internal/core/src/index/json_stats/JsonKeyStats.cpp index e8e3187e12..f0d9bdad58 100644 --- a/internal/core/src/index/json_stats/JsonKeyStats.cpp +++ b/internal/core/src/index/json_stats/JsonKeyStats.cpp @@ -1135,7 +1135,7 @@ JsonKeyStats::Load(milvus::tracer::TraceContext ctx, const Config& config) { load_priority_ = config[milvus::LOAD_PRIORITY]; LOG_INFO("load json stats for segment {} with load priority: {}", segment_id_, - load_priority_); + static_cast(load_priority_)); auto index_files = GetValueFromConfig>(config, "index_files"); diff --git a/internal/core/src/index/json_stats/utils.h b/internal/core/src/index/json_stats/utils.h index f6a1913c3a..da44dc036d 100644 --- a/internal/core/src/index/json_stats/utils.h +++ b/internal/core/src/index/json_stats/utils.h @@ -593,7 +593,7 @@ template <> struct fmt::formatter : fmt::formatter { template auto - format(const milvus::index::JSONType& jt, FormatContext& ctx) { + format(const milvus::index::JSONType& jt, FormatContext& ctx) const { switch (jt) { case milvus::index::JSONType::UNKNOWN: return fmt::format_to(ctx.out(), "UNKNOWN"); @@ -631,7 +631,8 @@ template <> struct fmt::formatter : fmt::formatter { auto - format(milvus::index::JsonKeyLayoutType type, fmt::format_context& ctx) { + format(milvus::index::JsonKeyLayoutType type, + fmt::format_context& ctx) const { std::string name; switch (type) { case milvus::index::JsonKeyLayoutType::TYPED: diff --git a/internal/core/src/mmap/ChunkVector.h b/internal/core/src/mmap/ChunkVector.h index 4e7ed6d693..c429c0916b 100644 --- a/internal/core/src/mmap/ChunkVector.h +++ b/internal/core/src/mmap/ChunkVector.h @@ -100,7 +100,7 @@ class ThreadSafeChunkVector : public ChunkVectorBase { AssertInfo(chunk_id < this->counter_, fmt::format("index out of range, index={}, counter_={}", chunk_id, - this->counter_)); + this->counter_.load())); if constexpr (!IsMmap || !IsVariableType) { auto ptr = (Type*)vec_[chunk_id].data(); AssertInfo( @@ -153,7 +153,7 @@ class ThreadSafeChunkVector : public ChunkVectorBase { AssertInfo(index < this->counter_, fmt::format("index out of range, index={}, counter_={}", index, - this->counter_)); + this->counter_.load())); return vec_[index].data(); } @@ -163,7 +163,7 @@ class ThreadSafeChunkVector : public ChunkVectorBase { AssertInfo(index < this->counter_, fmt::format("index out of range, index={}, counter_={}", index, - this->counter_)); + this->counter_.load())); return vec_[index].size(); } diff --git a/internal/core/src/mmap/ChunkVectorTest.cpp b/internal/core/src/mmap/ChunkVectorTest.cpp index a917fcb6d0..884b931ab4 100644 --- a/internal/core/src/mmap/ChunkVectorTest.cpp +++ b/internal/core/src/mmap/ChunkVectorTest.cpp @@ -22,7 +22,6 @@ using namespace milvus::segcore; using namespace milvus; -namespace pb = milvus::proto; class ChunkVectorTest : public ::testing::TestWithParam { public: void diff --git a/internal/core/src/query/PlanProto.cpp b/internal/core/src/query/PlanProto.cpp index fb5fb97ee9..a39be5e10e 100644 --- a/internal/core/src/query/PlanProto.cpp +++ b/internal/core/src/query/PlanProto.cpp @@ -112,7 +112,7 @@ ProtoParser::ParseSearchInfo(const planpb::VectorANNS& anns_proto) { // check if hints is valid ThrowInfo(ConfigInvalid, "hints: {} not supported", - search_info.search_params_[HINTS]); + search_info.search_params_[HINTS].dump()); } } } diff --git a/internal/core/src/rescores/Utils.h b/internal/core/src/rescores/Utils.h index d66b2d9f05..4a0f68ed0c 100644 --- a/internal/core/src/rescores/Utils.h +++ b/internal/core/src/rescores/Utils.h @@ -40,7 +40,7 @@ function_score_merge(const float& a, ThrowInfo(ErrorCode::UnexpectedError, fmt::format("unknown boost function mode: {}:{}", proto::plan::FunctionMode_Name(mode), - mode)); + static_cast(mode))); } } diff --git a/internal/core/src/segcore/SegmentGrowingIndexTest.cpp b/internal/core/src/segcore/SegmentGrowingIndexTest.cpp index ec0cf734d3..c698dfaba9 100644 --- a/internal/core/src/segcore/SegmentGrowingIndexTest.cpp +++ b/internal/core/src/segcore/SegmentGrowingIndexTest.cpp @@ -57,7 +57,6 @@ using namespace milvus; using namespace milvus::segcore; -namespace pb = milvus::proto; using Param = std::tuple(); diff --git a/internal/core/src/storage/ChunkManager.cpp b/internal/core/src/storage/ChunkManager.cpp index 287e6da3a8..0c21654d0f 100644 --- a/internal/core/src/storage/ChunkManager.cpp +++ b/internal/core/src/storage/ChunkManager.cpp @@ -120,19 +120,7 @@ GcpChunkManager::GcpChunkManager(const StorageConfig& storage_config) { remote_root_path_ = storage_config.root_path; if (storage_config.useIAM) { - sdk_options_.httpOptions.httpClientFactory_create_fn = []() { - auto client_factory = [](google::cloud::Options const& opts) - -> std::unique_ptr { - return google::cloud::rest_internal::MakeDefaultRestClient( - {}, opts); - }; - auto credentials = std::make_shared< - google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS:: - ComputeEngineCredentials>(google::cloud::Options{}, - client_factory); - return Aws::MakeShared( - GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials); - }; + ConfigureGoogleCloudIAMHttpClientFactory(sdk_options_); } InitSDKAPIDefault(storage_config.log_level); diff --git a/internal/core/src/storage/minio/MinioChunkManager.cpp b/internal/core/src/storage/minio/MinioChunkManager.cpp index 790da3deb7..2ec96ada5d 100644 --- a/internal/core/src/storage/minio/MinioChunkManager.cpp +++ b/internal/core/src/storage/minio/MinioChunkManager.cpp @@ -49,6 +49,7 @@ #include "common/Consts.h" #include "common/EasyAssert.h" #include "fmt/core.h" +#include "google/cloud/internal/rest_client.h" #include "google/cloud/storage/oauth2/compute_engine_credentials.h" #include "google/cloud/version.h" #include "log/Log.h" @@ -64,6 +65,22 @@ namespace milvus::storage { std::atomic MinioChunkManager::init_count_(0); std::mutex MinioChunkManager::client_mutex_; +void +ConfigureGoogleCloudIAMHttpClientFactory(Aws::SDKOptions& sdk_options) { + sdk_options.httpOptions.httpClientFactory_create_fn = []() { + auto credentials = + std::make_shared( + google::cloud::Options{}, + [](google::cloud::Options const& opts) { + return google::cloud::rest_internal::MakeDefaultRestClient( + "", opts); + }); + return Aws::MakeShared( + GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials); + }; +} + static void SwallowHandler(int signal) { #pragma GCC diagnostic push @@ -126,20 +143,7 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type, sigaddset(&psa.sa_mask, SIGPIPE); sigaction(SIGPIPE, &psa, 0); if (type == RemoteStorageType::GOOGLE_CLOUD && useIAM) { - sdk_options_.httpOptions.httpClientFactory_create_fn = []() { - auto client_factory = [](google::cloud::Options const& opts) - -> std::unique_ptr< - google::cloud::rest_internal::RestClient> { - return google::cloud::rest_internal::MakeDefaultRestClient( - {}, opts); - }; - auto credentials = std::make_shared< - google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS:: - ComputeEngineCredentials>(google::cloud::Options{}, - client_factory); - return Aws::MakeShared( - GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials); - }; + ConfigureGoogleCloudIAMHttpClientFactory(sdk_options_); } LOG_INFO("init aws with log level:{}", log_level_str); auto get_aws_log_level = [](const std::string& level_str) { diff --git a/internal/core/src/storage/minio/MinioChunkManager.h b/internal/core/src/storage/minio/MinioChunkManager.h index 6a4cd52b48..bf5fd554a4 100644 --- a/internal/core/src/storage/minio/MinioChunkManager.h +++ b/internal/core/src/storage/minio/MinioChunkManager.h @@ -70,6 +70,9 @@ enum class RemoteStorageType { ALIYUN_CLOUD = 2, }; +void +ConfigureGoogleCloudIAMHttpClientFactory(Aws::SDKOptions& sdk_options); + template static std::string S3ErrorMessage(const std::string& func, diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index c21f93f2e9..110d55c852 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -14,7 +14,7 @@ # Update KNOWHERE_VERSION for the first occurrence milvus_add_pkg_config("knowhere") set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "") -set( KNOWHERE_VERSION 3cade7e ) +set( KNOWHERE_VERSION b20270cf507d6d3fb943a9c1d89a4093762a7b53 ) set( GIT_REPOSITORY "https://github.com/zilliztech/knowhere.git") message(STATUS "Knowhere repo: ${GIT_REPOSITORY}") @@ -52,52 +52,6 @@ FetchContent_GetProperties( knowhere ) if ( NOT knowhere_POPULATED ) FetchContent_Populate( knowhere ) - # Patch knowhere cmake for macOS ARM64 - include hook.cc but exclude SVE sources - # SVE (Scalable Vector Extension) is not supported on Apple Silicon - # But hook.cc contains required symbols (fvec_hook, fvec_madd) needed for linking - if(APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") - set(_KNOWHERE_FAISS_CMAKE "${knowhere_SOURCE_DIR}/cmake/libs/libfaiss.cmake") - file(READ "${_KNOWHERE_FAISS_CMAKE}" _FAISS_CMAKE_CONTENT) - - # Verify patch targets exist (detect upstream changes early) - string(FIND "${_FAISS_CMAKE_CONTENT}" "set(UTILS_SRC src/simd/distances_ref.cc src/simd/distances_neon.cc)" _UTILS_POS) - if(_UTILS_POS EQUAL -1) - message(FATAL_ERROR "knowhere faiss patch failed: UTILS_SRC pattern not found in ${_KNOWHERE_FAISS_CMAKE}. " - "Upstream cmake may have changed (version: ${KNOWHERE_VERSION}).") - endif() - string(FIND "${_FAISS_CMAKE_CONTENT}" "set(UTILS_SVE_SRC src/simd/hook.cc src/simd/distances_sve.cc)" _SVE_POS) - if(_SVE_POS EQUAL -1) - message(FATAL_ERROR "knowhere faiss patch failed: UTILS_SVE_SRC pattern not found in ${_KNOWHERE_FAISS_CMAKE}. " - "Upstream cmake may have changed (version: ${KNOWHERE_VERSION}).") - endif() - string(FIND "${_FAISS_CMAKE_CONTENT}" "set(ALL_UTILS_SRC \${UTILS_SRC} \${UTILS_SVE_SRC})" _ALL_UTILS_POS) - if(_ALL_UTILS_POS EQUAL -1) - message(FATAL_ERROR "knowhere faiss patch failed: ALL_UTILS_SRC pattern not found in ${_KNOWHERE_FAISS_CMAKE}. " - "Upstream cmake may have changed (version: ${KNOWHERE_VERSION}).") - endif() - - # Step 1: Add hook.cc to UTILS_SRC for Apple (contains required symbols) - string(REPLACE - "set(UTILS_SRC src/simd/distances_ref.cc src/simd/distances_neon.cc)" - "set(UTILS_SRC src/simd/distances_ref.cc src/simd/distances_neon.cc src/simd/hook.cc)" - _FAISS_CMAKE_CONTENT "${_FAISS_CMAKE_CONTENT}") - - # Step 2: On Apple, exclude only distances_sve.cc from UTILS_SVE_SRC - string(REPLACE - "set(UTILS_SVE_SRC src/simd/hook.cc src/simd/distances_sve.cc)" - "set(UTILS_SVE_SRC src/simd/distances_sve.cc)" - _FAISS_CMAKE_CONTENT "${_FAISS_CMAKE_CONTENT}") - - # Step 3: Exclude SVE sources on Apple - string(REPLACE - "set(ALL_UTILS_SRC \${UTILS_SRC} \${UTILS_SVE_SRC})" - "if(APPLE)\n set(ALL_UTILS_SRC \${UTILS_SRC})\n message(STATUS \"Excluding SVE sources on Apple (hook.cc included in UTILS_SRC)\")\n else()\n set(ALL_UTILS_SRC \${UTILS_SRC} \${UTILS_SVE_SRC})\n endif()" - _FAISS_CMAKE_CONTENT "${_FAISS_CMAKE_CONTENT}") - - file(WRITE "${_KNOWHERE_FAISS_CMAKE}" "${_FAISS_CMAKE_CONTENT}") - message(STATUS "Patched knowhere cmake: hook.cc included, SVE excluded on Apple") - endif() - # Adding the following target: # knowhere add_subdirectory( ${knowhere_SOURCE_DIR} diff --git a/internal/core/thirdparty/milvus-common/CMakeLists.txt b/internal/core/thirdparty/milvus-common/CMakeLists.txt index 2c618f9300..9c6e683b8c 100644 --- a/internal/core/thirdparty/milvus-common/CMakeLists.txt +++ b/internal/core/thirdparty/milvus-common/CMakeLists.txt @@ -13,7 +13,7 @@ milvus_add_pkg_config("milvus-common") set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "") -set( MILVUS-COMMON-VERSION f192a19 ) +set( MILVUS-COMMON-VERSION e16257c98c8cbae7ca2d4b993838cdd95388fa62 ) set( GIT_REPOSITORY "https://github.com/zilliztech/milvus-common.git") message(STATUS "milvus-common repo: ${GIT_REPOSITORY}") @@ -32,7 +32,6 @@ FetchContent_Declare( GIT_TAG ${MILVUS-COMMON-VERSION} SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/milvus-common-src BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/milvus-common-build - SOURCE_SUBDIR cpp DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) FetchContent_GetProperties( milvus-common ) diff --git a/internal/core/thirdparty/milvus-storage/CMakeLists.txt b/internal/core/thirdparty/milvus-storage/CMakeLists.txt index 12b56b415f..5510c20b78 100644 --- a/internal/core/thirdparty/milvus-storage/CMakeLists.txt +++ b/internal/core/thirdparty/milvus-storage/CMakeLists.txt @@ -28,7 +28,29 @@ else () set(WITH_AZURE_FS OFF CACHE BOOL "" FORCE ) endif () -set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} ) +# Disable benchmark for milvus-storage (we don't need it and it may have missing deps) +set(WITH_BENCHMARK OFF CACHE BOOL "" FORCE) + +set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} ${CONAN_PROTOBUF_ROOT} ) + +# Make Conan's protoc visible to milvus-storage's find_package(Protobuf) +find_program(_PROTOC_PROGRAM protoc + PATHS ${CONAN_BIN_DIRS_PROTOBUF} ${CONAN_BIN_DIRS} + "${CMAKE_BINARY_DIR}/bin" + NO_DEFAULT_PATH) +if(NOT _PROTOC_PROGRAM) + file(GLOB _PROTOC_CANDIDATES + "${CONAN_BIN_DIRS_PROTOBUF}/protoc-*" + "${CMAKE_BINARY_DIR}/bin/protoc-*") + if(_PROTOC_CANDIDATES) + list(SORT _PROTOC_CANDIDATES) + list(GET _PROTOC_CANDIDATES -1 _PROTOC_PROGRAM) + endif() +endif() +if(_PROTOC_PROGRAM) + message(STATUS "Setting Protobuf_PROTOC_EXECUTABLE for milvus-storage: ${_PROTOC_PROGRAM}") + set(Protobuf_PROTOC_EXECUTABLE "${_PROTOC_PROGRAM}" CACHE FILEPATH "" FORCE) +endif() FetchContent_Declare( milvus-storage GIT_REPOSITORY ${GIT_REPOSITORY} @@ -42,60 +64,6 @@ FetchContent_GetProperties( milvus-storage ) if ( NOT milvus-storage_POPULATED ) FetchContent_Populate( milvus-storage ) - if(APPLE) - # On macOS, Conan's aws-c-* packages use deprecated Security framework APIs - # removed in macOS 15+. Instead, use Homebrew's aws-sdk-cpp (1.11.735+) - # which has the compatibility fixes. - # - # Strategy: create an IMPORTED target with the Conan target name - # (AWS::aws-sdk-cpp-identity-management) that wraps Homebrew's AWS SDK. - # This way milvus-storage's unmodified CMakeLists.txt links correctly. - execute_process( - COMMAND brew --prefix - OUTPUT_VARIABLE _HOMEBREW_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) - if(NOT _HOMEBREW_PREFIX) - set(_HOMEBREW_PREFIX "/opt/homebrew") - endif() - if(EXISTS "${_HOMEBREW_PREFIX}/lib/cmake/AWSSDK") - set(AWSSDK_DIR "${_HOMEBREW_PREFIX}/lib/cmake/AWSSDK") - else() - message(FATAL_ERROR "AWS SDK cmake config not found in Homebrew " - "(searched ${_HOMEBREW_PREFIX}/lib/cmake/AWSSDK). " - "Install with: brew install aws-sdk-cpp") - endif() - find_package(AWSSDK REQUIRED COMPONENTS core s3 identity-management) - - # aws-sdk-cpp 1.11.750 had a packaging bug: the core CMakeLists.txt - # was missing the install() rule for pagination headers, so - # Paginator.h is absent from the installation even though the STS - # module's STSPaginationBase.h references it. Homebrew backported - # the fix (commit 175e803) with rebuild 1. Fail early with a clear - # message so developers know to reinstall. - if(NOT EXISTS "${_HOMEBREW_PREFIX}/include/aws/core/utils/pagination/Paginator.h") - message(FATAL_ERROR - "aws/core/utils/pagination/Paginator.h is missing from " - "Homebrew's aws-sdk-cpp installation. This is a known " - "packaging bug in aws-sdk-cpp 1.11.750. " - "Fix with: brew reinstall aws-sdk-cpp") - endif() - - if(NOT TARGET AWS::aws-sdk-cpp-identity-management) - add_library(AWS::aws-sdk-cpp-identity-management INTERFACE IMPORTED) - set_target_properties(AWS::aws-sdk-cpp-identity-management PROPERTIES - INTERFACE_LINK_LIBRARIES "${AWSSDK_LINK_LIBRARIES}") - endif() - - # Ignore Homebrew paths so milvus-storage's find_package(Boost) uses - # Conan's Boost 1.83, not Homebrew's Boost 1.90+ (ABI incompatible). - set(_SAVED_CMAKE_IGNORE_PATH "${CMAKE_IGNORE_PATH}") - list(APPEND CMAKE_IGNORE_PATH - "/opt/homebrew" "/opt/homebrew/include" "/opt/homebrew/lib" - "/usr/local" "/usr/local/include" "/usr/local/lib") - endif() - # Adding the following target: # milvus-storage add_subdirectory( ${milvus-storage_SOURCE_DIR}/cpp @@ -104,8 +72,25 @@ if ( NOT milvus-storage_POPULATED ) # Enable S3 filesystem support on all platforms target_compile_definitions(milvus-storage PRIVATE MILVUS_S3_FS) + # milvus-storage uses Aws::S3::S3Client and Aws::S3::Model::* directly but only + # links AWS::aws-sdk-cpp-identity-management in its own CMakeLists.txt. The S3 + # symbols happen to resolve through Arrow's transitive dependency on aws-sdk-cpp-s3, + # but this is fragile (depends on Arrow package config). Explicitly link the S3 + # component so the symbols are always resolved in libmilvus-storage.so. + # Conan's cmake_find_package generator creates FindAWSSDK.cmake but it must be + # explicitly loaded to create the AWS::* targets. + find_package(AWSSDK REQUIRED) + if(TARGET AWS::aws-sdk-cpp-s3) + target_link_libraries(milvus-storage PUBLIC AWS::aws-sdk-cpp-s3) + elseif(TARGET AWS::s3) + target_link_libraries(milvus-storage PUBLIC AWS::s3) + else() + message(WARNING "Neither AWS::aws-sdk-cpp-s3 nor AWS::s3 target found after find_package(AWSSDK)") + endif() + + # On macOS, aws-c-io uses Apple's Network.framework for socket operations if(APPLE) - set(CMAKE_IGNORE_PATH "${_SAVED_CMAKE_IGNORE_PATH}") + target_link_libraries(milvus-storage PUBLIC "-framework Network") endif() endif() diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 08ce79ae81..6d2438eefe 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -123,10 +123,11 @@ target_link_libraries(all_tests knowhere milvus-storage milvus-common - milvus-planparser-cpp ) -# Add RPATH for plan parser library to find libmilvus-planparser.so at runtime +# Link plan parser library using full path and set RPATH +target_link_options(all_tests PRIVATE "-L${PLANPARSER_LIB_DIR}") +target_link_libraries(all_tests milvus-planparser-cpp) set_target_properties(all_tests PROPERTIES BUILD_RPATH "${PLANPARSER_LIB_DIR}" INSTALL_RPATH "${PLANPARSER_LIB_DIR}" @@ -161,4 +162,10 @@ target_link_libraries(bitset_test gtest ${CONAN_LIBS} ) +if (LINUX) + # folly shared library requires libaio; use --no-as-needed to ensure + # the linker pulls it in even when bitset_test doesn't reference aio directly + target_link_options(bitset_test PRIVATE "LINKER:--no-as-needed") + target_link_libraries(bitset_test aio) +endif() install(TARGETS bitset_test DESTINATION unittest) diff --git a/internal/core/unittest/init_gtest.cpp b/internal/core/unittest/init_gtest.cpp index a844a16a19..d27fe0f668 100644 --- a/internal/core/unittest/init_gtest.cpp +++ b/internal/core/unittest/init_gtest.cpp @@ -23,6 +23,13 @@ #include "exec/expression/function/init_c.h" #include "folly/init/Init.h" #include "milvus-storage/filesystem/fs.h" +#include "pb/cgo_msg.pb.h" +#include "pb/clustering.pb.h" +#include "pb/common.pb.h" +#include "pb/index_cgo_msg.pb.h" +#include "pb/plan.pb.h" +#include "pb/schema.pb.h" +#include "pb/segcore.pb.h" #include "storage/LocalChunkManagerSingleton.h" #include "storage/MmapManager.h" #include "storage/RemoteChunkManagerSingleton.h" @@ -38,6 +45,20 @@ main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); folly::Init follyInit(&argc, &argv, false); + // Force eager initialization of all protobuf descriptors. + // Protobuf 5.x uses lazy descriptor initialization via absl::call_once. + // Under ASAN, this lazy init can intermittently fail with + // "Check failed: file != nullptr" in AssignDescriptorsImpl. + // Initializing descriptors here (single-threaded, before any tests) + // eliminates the flaky failure. + ::milvus::proto::common::SegmentStats::descriptor(); + ::milvus::proto::schema::CollectionSchema::descriptor(); + ::milvus::proto::plan::PlanNode::descriptor(); + ::milvus::proto::segcore::RetrieveResults::descriptor(); + ::milvus::proto::indexcgo::TypeParams::descriptor(); + ::milvus::proto::cgo::SerializedIndexFileInfo::descriptor(); + ::milvus::proto::clustering::StorageConfig::descriptor(); + // Determine the base directory for test output. // Priority: MILVUS_TEST_ROOT_DIR env var > compile-time MILVUS_CPPUT_OUTPUT_DIR std::string base_dir; diff --git a/internal/core/unittest/test_query.cpp b/internal/core/unittest/test_query.cpp index 6d5915b7fb..95b66470e0 100644 --- a/internal/core/unittest/test_query.cpp +++ b/internal/core/unittest/test_query.cpp @@ -475,8 +475,8 @@ TEST(Query, InnerProduct) { } TEST(Query, DISABLED_FillSegment) { - namespace pb = milvus::proto; - pb::schema::CollectionSchema proto; + namespace milvus_pb = milvus::proto; + milvus_pb::schema::CollectionSchema proto; proto.set_name("col"); proto.set_description("asdfhsalkgfhsadg"); auto dim = 16; @@ -494,7 +494,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_is_primary_key(false); field->set_description("asdgfsagf"); field->set_fieldid(100); - field->set_data_type(pb::schema::DataType::FloatVector); + field->set_data_type(milvus_pb::schema::DataType::FloatVector); auto param = field->add_type_params(); param->set_key("dim"); param->set_value("16"); @@ -510,7 +510,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(101); field->set_is_primary_key(true); field->set_description("asdgfsagf"); - field->set_data_type(pb::schema::DataType::Int64); + field->set_data_type(milvus_pb::schema::DataType::Int64); } { @@ -520,7 +520,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(102); field->set_is_primary_key(false); field->set_description("asdgfsagf"); - field->set_data_type(pb::schema::DataType::Int32); + field->set_data_type(milvus_pb::schema::DataType::Int32); } auto schema = Schema::ParseFrom(proto); @@ -556,7 +556,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(103); field->set_is_primary_key(false); field->set_description("lack null binlog"); - field->set_data_type(pb::schema::DataType::Float); + field->set_data_type(milvus_pb::schema::DataType::Float); } { @@ -566,7 +566,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(104); field->set_is_primary_key(false); field->set_description("lack default value binlog"); - field->set_data_type(pb::schema::DataType::Bool); + field->set_data_type(milvus_pb::schema::DataType::Bool); field->mutable_default_value()->set_bool_data(bool_default_value); } @@ -577,7 +577,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(105); field->set_is_primary_key(false); field->set_description("lack default value binlog"); - field->set_data_type(pb::schema::DataType::Int32); + field->set_data_type(milvus_pb::schema::DataType::Int32); field->mutable_default_value()->set_int_data(int_default_value); } @@ -588,7 +588,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(106); field->set_is_primary_key(false); field->set_description("lack default value binlog"); - field->set_data_type(pb::schema::DataType::Int64); + field->set_data_type(milvus_pb::schema::DataType::Int64); field->mutable_default_value()->set_int_data(long_default_value); } @@ -599,7 +599,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(107); field->set_is_primary_key(false); field->set_description("lack default value binlog"); - field->set_data_type(pb::schema::DataType::Float); + field->set_data_type(milvus_pb::schema::DataType::Float); field->mutable_default_value()->set_float_data(float_default_value); } @@ -610,7 +610,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(108); field->set_is_primary_key(false); field->set_description("lack default value binlog"); - field->set_data_type(pb::schema::DataType::Double); + field->set_data_type(milvus_pb::schema::DataType::Double); field->mutable_default_value()->set_double_data(double_default_value); } @@ -621,7 +621,7 @@ TEST(Query, DISABLED_FillSegment) { field->set_fieldid(109); field->set_is_primary_key(false); field->set_description("lack default value binlog"); - field->set_data_type(pb::schema::DataType::VarChar); + field->set_data_type(milvus_pb::schema::DataType::VarChar); auto str_type_params = field->add_type_params(); str_type_params->set_key(MAX_LENGTH); str_type_params->set_value(std::to_string(64)); diff --git a/pkg/eventlog/event_log.pb.go b/pkg/eventlog/event_log.pb.go index 60318589ba..b1a784af0e 100644 --- a/pkg/eventlog/event_log.pb.go +++ b/pkg/eventlog/event_log.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: event_log.proto package eventlog diff --git a/pkg/eventlog/event_log_grpc.pb.go b/pkg/eventlog/event_log_grpc.pb.go index 1c72082770..1864b45790 100644 --- a/pkg/eventlog/event_log_grpc.pb.go +++ b/pkg/eventlog/event_log_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: event_log.proto package eventlog diff --git a/pkg/proto/cgopb/cgo_msg.pb.go b/pkg/proto/cgopb/cgo_msg.pb.go index 242fce2f1a..7c99ff231a 100644 --- a/pkg/proto/cgopb/cgo_msg.pb.go +++ b/pkg/proto/cgopb/cgo_msg.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: cgo_msg.proto package cgopb diff --git a/pkg/proto/clusteringpb/clustering.pb.go b/pkg/proto/clusteringpb/clustering.pb.go index 311c7e16c4..95759d3acf 100644 --- a/pkg/proto/clusteringpb/clustering.pb.go +++ b/pkg/proto/clusteringpb/clustering.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: clustering.proto package clusteringpb diff --git a/pkg/proto/datapb/data_coord.pb.go b/pkg/proto/datapb/data_coord.pb.go index f5e7125606..112f40c3b2 100644 --- a/pkg/proto/datapb/data_coord.pb.go +++ b/pkg/proto/datapb/data_coord.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: data_coord.proto package datapb diff --git a/pkg/proto/datapb/data_coord_grpc.pb.go b/pkg/proto/datapb/data_coord_grpc.pb.go index 9e6350f56f..8aac5ac48f 100644 --- a/pkg/proto/datapb/data_coord_grpc.pb.go +++ b/pkg/proto/datapb/data_coord_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: data_coord.proto package datapb diff --git a/pkg/proto/etcdpb/etcd_meta.pb.go b/pkg/proto/etcdpb/etcd_meta.pb.go index ba8a65f6ce..bdbca0fe31 100644 --- a/pkg/proto/etcdpb/etcd_meta.pb.go +++ b/pkg/proto/etcdpb/etcd_meta.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: etcd_meta.proto package etcdpb diff --git a/pkg/proto/indexcgopb/index_cgo_msg.pb.go b/pkg/proto/indexcgopb/index_cgo_msg.pb.go index 20f5afefbd..926d9178b5 100644 --- a/pkg/proto/indexcgopb/index_cgo_msg.pb.go +++ b/pkg/proto/indexcgopb/index_cgo_msg.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: index_cgo_msg.proto package indexcgopb diff --git a/pkg/proto/indexpb/index_coord.pb.go b/pkg/proto/indexpb/index_coord.pb.go index 8ea21af386..9bac951fc7 100644 --- a/pkg/proto/indexpb/index_coord.pb.go +++ b/pkg/proto/indexpb/index_coord.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: index_coord.proto package indexpb diff --git a/pkg/proto/indexpb/index_coord_grpc.pb.go b/pkg/proto/indexpb/index_coord_grpc.pb.go index 1daaaa7e1d..0432824be7 100644 --- a/pkg/proto/indexpb/index_coord_grpc.pb.go +++ b/pkg/proto/indexpb/index_coord_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: index_coord.proto package indexpb diff --git a/pkg/proto/internalpb/internal.pb.go b/pkg/proto/internalpb/internal.pb.go index 166ef21705..7cf04f9cd9 100644 --- a/pkg/proto/internalpb/internal.pb.go +++ b/pkg/proto/internalpb/internal.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: internal.proto package internalpb diff --git a/pkg/proto/messagespb/messages.pb.go b/pkg/proto/messagespb/messages.pb.go index 96efca5add..e73eac3f5c 100644 --- a/pkg/proto/messagespb/messages.pb.go +++ b/pkg/proto/messagespb/messages.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: messages.proto package messagespb diff --git a/pkg/proto/modelservicepb/model_service.pb.go b/pkg/proto/modelservicepb/model_service.pb.go index d347ce8371..8c43011e88 100644 --- a/pkg/proto/modelservicepb/model_service.pb.go +++ b/pkg/proto/modelservicepb/model_service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: model_service.proto package modelservicepb diff --git a/pkg/proto/modelservicepb/model_service_grpc.pb.go b/pkg/proto/modelservicepb/model_service_grpc.pb.go index 17e3a480c6..881f786b5b 100644 --- a/pkg/proto/modelservicepb/model_service_grpc.pb.go +++ b/pkg/proto/modelservicepb/model_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: model_service.proto package modelservicepb diff --git a/pkg/proto/planpb/plan.pb.go b/pkg/proto/planpb/plan.pb.go index 7975d44518..9713fb3cc9 100644 --- a/pkg/proto/planpb/plan.pb.go +++ b/pkg/proto/planpb/plan.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: plan.proto package planpb diff --git a/pkg/proto/proxypb/proxy.pb.go b/pkg/proto/proxypb/proxy.pb.go index c41a42459c..864ca5a31a 100644 --- a/pkg/proto/proxypb/proxy.pb.go +++ b/pkg/proto/proxypb/proxy.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: proxy.proto package proxypb diff --git a/pkg/proto/proxypb/proxy_grpc.pb.go b/pkg/proto/proxypb/proxy_grpc.pb.go index 3ba6adf440..12dabe1ef6 100644 --- a/pkg/proto/proxypb/proxy_grpc.pb.go +++ b/pkg/proto/proxypb/proxy_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: proxy.proto package proxypb diff --git a/pkg/proto/querypb/query_coord.pb.go b/pkg/proto/querypb/query_coord.pb.go index 1d2baaf2bc..b6cb097c5c 100644 --- a/pkg/proto/querypb/query_coord.pb.go +++ b/pkg/proto/querypb/query_coord.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: query_coord.proto package querypb diff --git a/pkg/proto/querypb/query_coord_grpc.pb.go b/pkg/proto/querypb/query_coord_grpc.pb.go index 45a61b755d..448bb6f386 100644 --- a/pkg/proto/querypb/query_coord_grpc.pb.go +++ b/pkg/proto/querypb/query_coord_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: query_coord.proto package querypb diff --git a/pkg/proto/rootcoordpb/root_coord.pb.go b/pkg/proto/rootcoordpb/root_coord.pb.go index 2a300107cc..b14fdc5ddb 100644 --- a/pkg/proto/rootcoordpb/root_coord.pb.go +++ b/pkg/proto/rootcoordpb/root_coord.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: root_coord.proto package rootcoordpb diff --git a/pkg/proto/rootcoordpb/root_coord_grpc.pb.go b/pkg/proto/rootcoordpb/root_coord_grpc.pb.go index fc334bee04..881c9e66fd 100644 --- a/pkg/proto/rootcoordpb/root_coord_grpc.pb.go +++ b/pkg/proto/rootcoordpb/root_coord_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: root_coord.proto package rootcoordpb diff --git a/pkg/proto/segcorepb/segcore.pb.go b/pkg/proto/segcorepb/segcore.pb.go index 3f1d73b273..338326d478 100644 --- a/pkg/proto/segcorepb/segcore.pb.go +++ b/pkg/proto/segcorepb/segcore.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: segcore.proto package segcorepb diff --git a/pkg/proto/streamingpb/streaming.pb.go b/pkg/proto/streamingpb/streaming.pb.go index c4276f7b3c..3cde0e6e98 100644 --- a/pkg/proto/streamingpb/streaming.pb.go +++ b/pkg/proto/streamingpb/streaming.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: streaming.proto package streamingpb diff --git a/pkg/proto/streamingpb/streaming_grpc.pb.go b/pkg/proto/streamingpb/streaming_grpc.pb.go index d2799d12f9..c8a623d333 100644 --- a/pkg/proto/streamingpb/streaming_grpc.pb.go +++ b/pkg/proto/streamingpb/streaming_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: streaming.proto package streamingpb diff --git a/pkg/proto/workerpb/worker.pb.go b/pkg/proto/workerpb/worker.pb.go index b1ec5d0788..fe473fc01e 100644 --- a/pkg/proto/workerpb/worker.pb.go +++ b/pkg/proto/workerpb/worker.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.33.0 -// protoc v3.21.4 +// protoc v5.27.0 // source: worker.proto package workerpb diff --git a/pkg/proto/workerpb/worker_grpc.pb.go b/pkg/proto/workerpb/worker_grpc.pb.go index 0a7b202e8d..d97c409e84 100644 --- a/pkg/proto/workerpb/worker_grpc.pb.go +++ b/pkg/proto/workerpb/worker_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.4 +// - protoc v5.27.0 // source: worker.proto package workerpb diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh index 16d300d8b7..17da8b7370 100644 --- a/scripts/3rdparty_build.sh +++ b/scripts/3rdparty_build.sh @@ -88,11 +88,39 @@ if [[ -f "$HOME/.local/bin/conan" ]]; then export PATH="$HOME/.local/bin:$PATH" fi +# Ensure conan version matches the required version (1.66.0) +# CI Docker images may have an older version pre-installed +REQUIRED_CONAN_VERSION="1.66.0" +CURRENT_CONAN_VERSION=$(conan --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "0.0.0") +if [[ "${CURRENT_CONAN_VERSION}" != "${REQUIRED_CONAN_VERSION}" ]]; then + echo "Conan version mismatch: ${CURRENT_CONAN_VERSION} != ${REQUIRED_CONAN_VERSION}, upgrading..." + pip3 install conan==${REQUIRED_CONAN_VERSION} 2>/dev/null || pip install conan==${REQUIRED_CONAN_VERSION} +fi + pushd ${BUILD_OUTPUT_DIR} +# Unset LD_PRELOAD and LD_LIBRARY_PATH to prevent jemalloc and other shared +# libraries (set by setenv.sh) from being injected into conan's Python process. +# - LD_PRELOAD with jemalloc causes immediate segfaults in Python. +# - LD_LIBRARY_PATH with cmake_build/lib/ (populated by conan imports on +# previous runs) causes the Python ssl module to load a mismatched +# libssl.so during the conan update check, also triggering segfaults. +# The pre_build hook (below) handles setting LD_LIBRARY_PATH per-build +# for tools like grpc_cpp_plugin that need shared libs at build time. +SAVED_LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}" +unset LD_PRELOAD +unset LD_LIBRARY_PATH + export CONAN_REVISIONS_ENABLED=1 export CXXFLAGS="-Wno-error=address -Wno-error=deprecated-declarations -include cstdint" export CFLAGS="-Wno-error=address -Wno-error=deprecated-declarations" +# LLVM from Homebrew doesn't set TARGET_OS_OSX=1 (unlike Apple's clang), which causes +# macOS SDK headers to exclude macOS-specific APIs (SecImportExport, SCPreferences, etc.). +# This fixes aws-c-cal, c-ares, and other packages that depend on macOS Security/SystemConfig APIs. +if [[ "$(uname -s)" == "Darwin" ]]; then + export CFLAGS="${CFLAGS} -DTARGET_OS_OSX=1" + export CXXFLAGS="${CXXFLAGS} -DTARGET_OS_OSX=1" +fi # Allow CMake 4.x to build packages with old cmake_minimum_required versions (< 3.5) export CMAKE_POLICY_VERSION_MINIMUM=3.5 @@ -103,6 +131,51 @@ if [[ ! `conan remote list` == *default-conan-local* ]]; then conan remote add default-conan-local $CONAN_ARTIFACTORY_URL fi +# Install a conan pre_build hook that sets LD_LIBRARY_PATH (Linux) or +# DYLD_LIBRARY_PATH (macOS) before each package build. This ensures that +# tools like grpc_cpp_plugin can find shared libraries (e.g. libprotoc.so) +# when invoked during downstream package builds (opentelemetry-cpp, googleapis). +# Use "conan config home" to get the correct conan home directory, as ~ may +# differ from the conan home in CI containers. +CONAN_HOME_DIR=$(conan config home 2>/dev/null || echo "${CONAN_USER_HOME:-$HOME}/.conan") +mkdir -p "$CONAN_HOME_DIR/hooks" +cat > "$CONAN_HOME_DIR/hooks/fix_shared_lib_env.py" << 'HOOK_EOF' +import os, platform + +_saved_env = {} + +def pre_build(output, conanfile, **kwargs): + """Set library path before build so tools like grpc_cpp_plugin can find + shared libs (e.g. libprotoc.so). Saved state is restored in post_build + to avoid polluting the conan process environment.""" + dep_lib_dirs = [] + try: + for _, dep in conanfile.deps_cpp_info.dependencies: + for p in dep.lib_paths: + if os.path.isdir(p): + dep_lib_dirs.append(p) + except Exception: + return + if not dep_lib_dirs: + return + env_var = "DYLD_LIBRARY_PATH" if platform.system() == "Darwin" else "LD_LIBRARY_PATH" + _saved_env[env_var] = os.environ.get(env_var) + existing = os.environ.get(env_var, "") + new_val = ":".join(dep_lib_dirs) + (":" + existing if existing else "") + os.environ[env_var] = new_val + output.info("Set %s for %d dependency lib dirs" % (env_var, len(dep_lib_dirs))) + +def post_build(output, conanfile, **kwargs): + """Restore library path after build to prevent conan process corruption.""" + for env_var, old_val in _saved_env.items(): + if old_val is None: + os.environ.pop(env_var, None) + else: + os.environ[env_var] = old_val + _saved_env.clear() +HOOK_EOF +conan config set hooks.fix_shared_lib_env 2>/dev/null + unameOut="$(uname -s)" case "${unameOut}" in Darwin*) @@ -111,7 +184,57 @@ case "${unameOut}" in export CMAKE_CXX_COMPILER_LAUNCHER=ccache echo "Using CXX: $CXX" echo "Using CC: $CC" - conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; } + CONAN_ARGS="--install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -u" + + # On macOS, Conan packages with shared libraries (protobuf, grpc) produce + # binaries (protoc, grpc_cpp_plugin) whose install rpaths don't include + # dependency lib dirs (e.g. abseil). This causes dyld failures when + # downstream packages (opentelemetry-cpp, googleapis) invoke them. + # + # Fix: install a Conan post_package hook that adds dependency rpaths to + # each package's binaries right after it is built (before the manifest is + # sealed), so every tool binary works the first time it is invoked. + mkdir -p "$CONAN_HOME_DIR/hooks" + conan config set hooks.fix_macos_rpaths 2>/dev/null + cat > "$CONAN_HOME_DIR/hooks/fix_macos_rpaths.py" << 'HOOK_EOF' +import os, platform, subprocess, stat + +def post_package(output, conanfile, conanfile_path, **kwargs): + if platform.system() != "Darwin": + return + pkg = conanfile.package_folder + if not pkg: + return + bin_dir = os.path.join(pkg, "bin") + if not os.path.isdir(bin_dir): + return + dep_lib_dirs = [] + try: + for _, dep in conanfile.deps_cpp_info.dependencies: + for p in dep.lib_paths: + if os.path.isdir(p): + dep_lib_dirs.append(p) + except Exception: + return + if not dep_lib_dirs: + return + fixed = [] + for name in os.listdir(bin_dir): + fp = os.path.join(bin_dir, name) + if not os.path.isfile(fp) or not (os.stat(fp).st_mode & stat.S_IXUSR): + continue + r = subprocess.run(["file", fp], capture_output=True, text=True) + if "Mach-O" not in r.stdout: + continue + for lib_path in dep_lib_dirs: + subprocess.run(["install_name_tool", "-add_rpath", lib_path, fp], + capture_output=True) + fixed.append(name) + if fixed: + output.info("Fixed macOS rpaths for: %s" % ", ".join(fixed)) +HOOK_EOF + + conan install ${CPP_SRC_DIR} ${CONAN_ARGS} || { echo 'conan install failed'; exit 1; } ;; Linux*) if [ -f /etc/os-release ]; then @@ -123,9 +246,9 @@ case "${unameOut}" in export CPU_TARGET=avx GCC_VERSION=`gcc -dumpversion` if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then - conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -u || { echo 'conan install failed'; exit 1; } + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; } else - conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -u || { echo 'conan install failed'; exit 1; } + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; } fi ;; *) @@ -133,6 +256,11 @@ case "${unameOut}" in ;; esac +# Restore LD_LIBRARY_PATH so downstream build steps can find shared libs +if [[ -n "${SAVED_LD_LIBRARY_PATH}" ]]; then + export LD_LIBRARY_PATH="${SAVED_LD_LIBRARY_PATH}" +fi + popd mkdir -p ${ROOT_DIR}/internal/core/output/lib diff --git a/scripts/README.md b/scripts/README.md index 72044c0516..1d92b07418 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,7 +6,7 @@ OS: Ubuntu 20.04 go：1.21 cmake: >=3.18 -gcc： 7.5 +gcc: >= 11 ``` ## Install dependencies diff --git a/scripts/check_proto_product.sh b/scripts/check_proto_product.sh index 18469937fa..61342984ab 100755 --- a/scripts/check_proto_product.sh +++ b/scripts/check_proto_product.sh @@ -29,7 +29,9 @@ if [[ $(uname -s) == "Darwin" ]]; then export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH" fi -check_result=$(git status | grep -E ".*pb.go|.*pb.cc|.*pb.h") +# Use -I to ignore hunks where all changed lines are protoc version comments. +# This avoids false failures when CI protoc version differs from the committer's. +check_result=$(git diff -I '//.*protoc' --name-only -- '*.pb.go' '*.pb.cc' '*.pb.h') echo "check_result: $check_result" if test -z "$check_result"; then exit 0 diff --git a/scripts/core_build.sh b/scripts/core_build.sh index ddc253f56c..80f72d8b97 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -254,15 +254,13 @@ echo "CC $CC" echo ${CMAKE_CMD} ${CMAKE_CMD} -G "${CMAKE_GENERATOR}" -# Export PROTOC for Rust prost-build (used by lance-encoding). -# Conan imports() copies the protoc binary into cmake_build/bin/. -# We prefer that over a system protoc to match the Conan protobuf version. -if [ -z "${PROTOC}" ] && [ -x "${BUILD_OUTPUT_DIR}/bin/protoc" ]; then - export PROTOC="${BUILD_OUTPUT_DIR}/bin/protoc" - echo "Using Conan protoc: ${PROTOC}" -elif [ -z "${PROTOC}" ] && command -v protoc &> /dev/null; then - export PROTOC="$(command -v protoc)" - echo "Using system protoc: ${PROTOC}" +# Export PROTOC for Rust crates (e.g. lance-encoding) that need it at build time +if [ -z "$PROTOC" ]; then + _PROTOC=$(grep -m1 "^Protobuf_PROTOC_EXECUTABLE" CMakeCache.txt 2>/dev/null | cut -d= -f2-) + if [ -n "$_PROTOC" ] && [ -f "$_PROTOC" ]; then + export PROTOC="$_PROTOC" + echo "Exported PROTOC=$PROTOC for Rust builds" + fi fi if [[ ${RUN_CPPLINT} == "ON" ]]; then diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh index 43ab5e671c..8562158d3e 100755 --- a/scripts/install_deps.sh +++ b/scripts/install_deps.sh @@ -21,12 +21,12 @@ # Supported platforms: # - macOS 12, 13, 14, 15 (Intel and Apple Silicon) # - Ubuntu 20.04, 22.04, 24.04 -# - Rocky Linux 8, 9 +# - Rocky Linux 9 # - Amazon Linux 2023 # # Compiler requirements: -# - macOS: LLVM/Clang 14-18 -# - Linux: GCC 9-14 +# - macOS: LLVM/Clang 15-17 +# - Linux: GCC 11-14 # # Usage: # ./scripts/install_deps.sh @@ -57,7 +57,7 @@ print_error() { # Minimum version requirements MIN_CMAKE_VERSION="3.26" MIN_GO_VERSION="1.21" -CONAN_VERSION="1.64.1" +CONAN_VERSION="1.66.0" RUST_VERSION="1.89" ####################################### @@ -198,11 +198,6 @@ install_mac_deps() { if [[ "$arch" == "arm64" ]]; then print_info "Installing Apple Silicon specific dependencies..." brew install --quiet openssl librdkafka - # AWS SDK for S3 support on macOS (needed for milvus-storage) - brew install --quiet aws-sdk-cpp - else - # Intel Mac - brew install --quiet aws-sdk-cpp fi # Create symlink for LLVM (for scripts that expect /usr/local/opt/llvm) @@ -328,60 +323,26 @@ install_rocky_deps() { local rocky_version=$(detect_rocky_version) print_info "Detected Rocky Linux ${rocky_version}" - # Enable EPEL repository - sudo dnf install -y epel-release + if [ "$rocky_version" -lt 9 ]; then + print_error "Rocky Linux ${rocky_version} is no longer supported. Please upgrade to Rocky Linux 9+." + exit 1 + fi - # Base packages - local base_packages=( - wget curl which git make ninja-build - automake python3-devel python3-pip - openblas-devel libaio libuuid-devel + # Enable EPEL and CRB repositories + sudo dnf install -y epel-release dnf-plugins-core + sudo dnf config-manager --set-enabled crb + + # Rocky 9 ships with GCC 11 by default — no toolset needed + print_info "Installing GCC and build dependencies for Rocky ${rocky_version}..." + sudo dnf install -y \ + gcc gcc-c++ gcc-gfortran \ + wget curl which git make ninja-build \ + automake python3-devel python3-pip \ + openblas-devel libaio libuuid-devel \ zip unzip ccache lcov libtool m4 autoconf - ) - # Version-specific packages - case "$rocky_version" in - 8) - print_info "Installing GCC toolset for Rocky 8..." - sudo dnf install -y gcc-toolset-12 gcc-toolset-12-libatomic-devel gcc-toolset-12-libstdc++-devel - sudo dnf install -y "${base_packages[@]}" - sudo dnf install -y atlas-devel - - # Enable GCC toolset - echo "source /opt/rh/gcc-toolset-12/enable" | sudo tee /etc/profile.d/gcc-toolset-12.sh - source /opt/rh/gcc-toolset-12/enable - - # Create libstdc++.a symlink for static linking (needed by conan packages like ninja) - local gcc12_libstdcxx="/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/libstdc++.a" - local system_gcc_dir="/usr/lib/gcc/x86_64-redhat-linux/8" - if [ -f "$gcc12_libstdcxx" ] && [ -d "$system_gcc_dir" ] && [ ! -f "$system_gcc_dir/libstdc++.a" ]; then - print_info "Creating libstdc++.a symlink for static linking..." - sudo ln -s "$gcc12_libstdcxx" "$system_gcc_dir/libstdc++.a" - fi - - # Install LLVM toolset for clang-format - sudo dnf install -y llvm-toolset - echo "export CLANG_TOOLS_PATH=/usr/bin" | sudo tee -a /etc/profile.d/llvm-toolset.sh - ;; - 9) - print_info "Installing GCC for Rocky 9..." - sudo dnf install -y gcc gcc-c++ gcc-gfortran - sudo dnf install -y "${base_packages[@]}" - - # Optionally install newer GCC toolset - sudo dnf install -y gcc-toolset-13 || true - if [ -d /opt/rh/gcc-toolset-13 ]; then - echo "source /opt/rh/gcc-toolset-13/enable" | sudo tee /etc/profile.d/gcc-toolset-13.sh - fi - - # Install clang-tools - sudo dnf install -y clang-tools-extra || sudo dnf install -y clang - ;; - *) - print_warn "Rocky Linux ${rocky_version} not explicitly supported" - sudo dnf install -y gcc gcc-c++ gcc-gfortran "${base_packages[@]}" - ;; - esac + # Install clang-tools for formatting + sudo dnf install -y clang-tools-extra || sudo dnf install -y clang # Install CMake if needed install_cmake_linux @@ -397,6 +358,7 @@ install_rocky_deps() { conan profile new default --detect --force 2>/dev/null || true conan profile update settings.compiler.version="$gcc_ver" default conan profile update settings.compiler.libcxx=libstdc++11 default + conan profile update settings.compiler.cppstd=17 default fi # Install Rust diff --git a/scripts/install_deps_msys.sh b/scripts/install_deps_msys.sh index 334aad5c55..5756b548f6 100644 --- a/scripts/install_deps_msys.sh +++ b/scripts/install_deps_msys.sh @@ -23,7 +23,7 @@ pacmanInstall() mingw-w64-x86_64-diffutils \ mingw-w64-x86_64-go - pip3 install conan==1.64.1 + pip3 install conan==1.66.0 } updateKey() diff --git a/scripts/run_cpp_codecov.sh b/scripts/run_cpp_codecov.sh index b38b0692a3..71c4249818 100755 --- a/scripts/run_cpp_codecov.sh +++ b/scripts/run_cpp_codecov.sh @@ -31,6 +31,12 @@ source ${ROOT_DIR}/scripts/setenv.sh MILVUS_CORE_DIR="${ROOT_DIR}/internal/core" MILVUS_CORE_UNITTEST_DIR="${MILVUS_CORE_DIR}/output/unittest" +# Suppress known LeakSanitizer false positives from uninstrumented shared libraries +LSAN_SUPPRESSIONS="${MILVUS_CORE_DIR}/lsan_suppressions.txt" +if [ -f "${LSAN_SUPPRESSIONS}" ]; then + export LSAN_OPTIONS="suppressions=${LSAN_SUPPRESSIONS}" +fi + echo "ROOT_DIR = ${ROOT_DIR}" echo "MILVUS_CORE_DIR = ${MILVUS_CORE_DIR}" echo "MILVUS_CORE_UNITTEST_DIR = ${MILVUS_CORE_UNITTEST_DIR}" @@ -67,6 +73,10 @@ for test in $(ls ${MILVUS_CORE_UNITTEST_DIR}); do # run unittest if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then echo "ASAN is enabled with env MILVUS_ENABLE_ASAN_LIB, set {$MILVUS_ENABLE_ASAN_LIB} at the front of LD_PRELOAD" + # protobuf 5.x RepeatedField triggers ASAN container-overflow false positives during static init + # LSAN suppression: __cxa_allocate_exception false positives from uninstrumented shared libs + ASAN_OPTIONS="${ASAN_OPTIONS:+${ASAN_OPTIONS}:}detect_container_overflow=0" \ + LSAN_OPTIONS="${LSAN_OPTIONS:+${LSAN_OPTIONS}:}suppressions=${MILVUS_CORE_DIR}/lsan_suppressions.txt" \ LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test} else ${MILVUS_CORE_UNITTEST_DIR}/${test} diff --git a/scripts/run_cpp_unittest.sh b/scripts/run_cpp_unittest.sh index 03df6f84f3..7e3c9efb3d 100755 --- a/scripts/run_cpp_unittest.sh +++ b/scripts/run_cpp_unittest.sh @@ -35,6 +35,12 @@ if [ -d "${CORE_INSTALL_PREFIX}/lib" ]; then export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${CORE_INSTALL_PREFIX}/lib fi +# Suppress known LeakSanitizer false positives from uninstrumented shared libraries +LSAN_SUPPRESSIONS="${MILVUS_CORE_DIR}/lsan_suppressions.txt" +if [ -f "${LSAN_SUPPRESSIONS}" ]; then + export LSAN_OPTIONS="suppressions=${LSAN_SUPPRESSIONS}" +fi + # run unittest arg="$1" filter_value="${arg#*=}" diff --git a/scripts/run_cpp_ut.sh b/scripts/run_cpp_ut.sh index 419b34ba30..129d5207fa 100755 --- a/scripts/run_cpp_ut.sh +++ b/scripts/run_cpp_ut.sh @@ -47,6 +47,12 @@ source ${ROOT_DIR}/scripts/setenv.sh MILVUS_CORE_DIR="${ROOT_DIR}/internal/core" MILVUS_CORE_UNITTEST_DIR="${MILVUS_CORE_DIR}/output/unittest" +# Suppress known LeakSanitizer false positives from uninstrumented shared libraries +LSAN_SUPPRESSIONS="${MILVUS_CORE_DIR}/lsan_suppressions.txt" +if [ -f "${LSAN_SUPPRESSIONS}" ]; then + export LSAN_OPTIONS="suppressions=${LSAN_SUPPRESSIONS}" +fi + echo "==============================================" echo "=== C++ Unit Tests (No Coverage) ===" echo "==============================================" @@ -85,7 +91,11 @@ run_test() { if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then echo "ASAN is enabled with env MILVUS_ENABLE_ASAN_LIB" - env $env_vars LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test} + # protobuf 5.x RepeatedField triggers ASAN container-overflow false positives during static init + # LSAN suppression: __cxa_allocate_exception false positives from uninstrumented shared libs + env $env_vars ASAN_OPTIONS="${ASAN_OPTIONS:+${ASAN_OPTIONS}:}detect_container_overflow=0" \ + LSAN_OPTIONS="${LSAN_OPTIONS:+${LSAN_OPTIONS}:}suppressions=${MILVUS_CORE_DIR}/lsan_suppressions.txt" \ + LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test} else env $env_vars ${MILVUS_CORE_UNITTEST_DIR}/${test} fi diff --git a/scripts/run_go_codecov.sh b/scripts/run_go_codecov.sh index ef7beb2dd8..b08aa0370e 100755 --- a/scripts/run_go_codecov.sh +++ b/scripts/run_go_codecov.sh @@ -56,6 +56,7 @@ for d in cmd/tools internal pkg client; do -coverprofile="$PROFILE" \ -covermode=atomic \ -p "$PARALLEL" \ + -ldflags="-r ${RPATH}" \ ./... if [ -f "$PROFILE" ]; then diff --git a/scripts/run_intergration_test.sh b/scripts/run_intergration_test.sh index 529837baf2..207348d050 100755 --- a/scripts/run_intergration_test.sh +++ b/scripts/run_intergration_test.sh @@ -45,8 +45,9 @@ TEST_CMD_WITH_ARGS=( -coverpkg=./... -coverprofile=profile.out -covermode=atomic - -caseTimeout=20m + -caseTimeout=20m -timeout=60m + -ldflags="-r ${RPATH}" ) function test_cmd() { diff --git a/scripts/setenv.sh b/scripts/setenv.sh index 9c0555c76d..2c5e76d829 100644 --- a/scripts/setenv.sh +++ b/scripts/setenv.sh @@ -51,20 +51,26 @@ case "${unameOut}" in echo "WARN: Cannot find $LIBJEMALLOC" fi export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:+$PKG_CONFIG_PATH:}$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64" - export RPATH=$LD_LIBRARY_PATH;; + MILVUS_LIB_DIRS="$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64:$ROOT_DIR/cmake_build/lib" + # LIBRARY_PATH: build-time linker search path (gcc/ld) + export LIBRARY_PATH="${LIBRARY_PATH:+$LIBRARY_PATH:}$MILVUS_LIB_DIRS" + # LD_LIBRARY_PATH: runtime linker search path - needed for transitive + # shared library dependencies (e.g. libfolly_exception_tracer.so has no + # RPATH and depends on libfolly_exception_tracer_base.so) + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$MILVUS_LIB_DIRS" + export RPATH=$MILVUS_LIB_DIRS;; Darwin*) - # detect llvm version by valid list (supports LLVM 14-17) + # detect llvm version by valid list (supports LLVM 15-17) # Note: LLVM 18 is NOT supported because Conan 1.x cannot handle the newer # compiler profiles/settings that LLVM 18 requires. Until we migrate to Conan 2, - # please use LLVM 17 or earlier. - for llvm_version in 17 16 15 14 NOT_FOUND ; do + # please use LLVM 15-17. + for llvm_version in 17 16 15 NOT_FOUND ; do if brew ls --versions llvm@${llvm_version} > /dev/null 2>&1; then break fi done if [ "${llvm_version}" = "NOT_FOUND" ] ; then - echo "ERROR: Valid LLVM (14-17) not installed. Run: brew install llvm@17" + echo "ERROR: Valid LLVM (15-17) not installed. Run: brew install llvm@17" echo "NOTE: LLVM 18 is not supported due to Conan 1.x incompatibility." exit 1 fi @@ -74,14 +80,35 @@ case "${unameOut}" in export CXX=${llvm_prefix}/bin/clang++ export ASM=${llvm_prefix}/bin/clang macos_sdk_path="$(xcrun --show-sdk-path)" - export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include -isysroot ${macos_sdk_path}" + # LLVM from Homebrew doesn't set TARGET_OS_OSX=1 (unlike Apple's clang), causing + # macOS SDK headers to exclude macOS-specific APIs. Define it explicitly. + export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include -isysroot ${macos_sdk_path} -DTARGET_OS_OSX=1" export CXXFLAGS=${CFLAGS} - export LDFLAGS="-L$(brew --prefix libomp)/lib" + # Include LLVM's own libc++/libc++abi so Conan packages that reference + # Apple-extension symbols (e.g. ___cxa_decrement_exception_refcount in + # folly) are resolved against the same runtime they were built with. + # -lc++abi is explicit because Homebrew LLVM clang++ does not inject it + # automatically (unlike Apple clang), but folly's static archive references it. + export LDFLAGS="-L$(brew --prefix libomp)/lib -L${llvm_prefix}/lib/c++ -lc++abi" + # Rust cc-rs crate uses the Xcode SDK version as deployment target if + # MACOSX_DEPLOYMENT_TARGET is unset. When the SDK version (e.g. 26.2) is + # newer than what this LLVM version understands, compilation fails. + # Cap it at the current macOS major version to stay compatible. + sdk_ver="$(xcrun --show-sdk-version 2>/dev/null)" + os_major="$(sw_vers -productVersion | cut -d. -f1)" + if [ -n "${sdk_ver}" ] && [ -n "${os_major}" ] && \ + [ "$(echo "${sdk_ver}" | cut -d. -f1)" -gt "${os_major}" ]; then + export MACOSX_DEPLOYMENT_TARGET="${os_major}.0" + fi + # Rust cc-rs needs TARGET_OS_OSX for correct platform detection in aws-lc-sys. + # Set for both ARM and Intel targets so cross-compilation also works. + export CFLAGS_aarch64_apple_darwin="-DTARGET_OS_OSX=1" + export CFLAGS_x86_64_apple_darwin="-DTARGET_OS_OSX=1" export CGO_CFLAGS="${CFLAGS}" export CGO_LDFLAGS="${LDFLAGS} -framework Security -framework CoreFoundation" - export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:+$PKG_CONFIG_PATH:}$ROOT_DIR/internal/core/output/lib/pkgconfig" - export DYLD_LIBRARY_PATH=$ROOT_DIR/internal/core/output/lib + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig" + export DYLD_LIBRARY_PATH=$ROOT_DIR/cmake_build/lib:$ROOT_DIR/internal/core/output/lib export RPATH=$DYLD_LIBRARY_PATH;; MINGW*) extra_path=$(cygpath -w "$ROOT_DIR/internal/core/output/lib") @@ -91,4 +118,3 @@ case "${unameOut}" in *) echo "does not supported" esac -