furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-20 09:45:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix macOS 15 ARM64 compilation issues (#7437) (#47810)

Browse Source 
issue: #47809

---------

Signed-off-by: xiaofanluan <xiaofan.luan@zilliz.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Xiaofan
2026-03-06 23:19:21 +08:00
committed by GitHub
co-authored by Claude Opus 4.6
parent d88c5655e1
commit d69bdd288c
92 changed files with 653 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/macos-cache-restore/action.yaml
+6 -6
View File
@@ -20,8 +20,8 @@ runs:
uses: actions/cache/restore@v4
with:
path: /var/tmp/ccache
key: macos15-ccache-${{ env.corehash }}
restore-keys: macos15-ccache-
key: macos15-arm-ccache-${{ env.corehash }}
restore-keys: macos15-arm-ccache-
- name: Mac Cache Go Mod Volumes
id: go-mod-cache
run: |
@@ -35,8 +35,8 @@ runs:
uses: actions/cache/restore@v4
with:
path: ~/go/pkg/mod
key: macos15-go-mod-${{ steps.go-mod-cache.outputs.hash }}
restore-keys: macos15-go-mod-
key: macos15-arm-go-mod-${{ steps.go-mod-cache.outputs.hash }}
restore-keys: macos15-arm-go-mod-
- name: Mac Cache Conan Packages
id: conan-cache
run: |
@@ -50,5 +50,5 @@ runs:
uses: actions/cache/restore@v4
with:
path: ~/.conan
key: macos15-conan-${{ steps.conan-cache.outputs.hash }}
restore-keys: macos15-conan-
key: macos15-arm-conan-${{ steps.conan-cache.outputs.hash }}
restore-keys: macos15-arm-conan-
.github/actions/macos-cache-save/action.yaml
+3 -3
View File
@@ -29,7 +29,7 @@ runs:
uses: actions/cache/save@v4
with:
path: /var/tmp/ccache
key: macos15-ccache-${{ env.corehash }}
key: macos15-arm-ccache-${{ env.corehash }}
- name: Generate Go Mod Hash
id: go-mod-hash
run: |
@@ -43,7 +43,7 @@ runs:
uses: actions/cache/save@v4
with:
path: ~/go/pkg/mod
key: macos15-go-mod-${{ steps.go-mod-hash.outputs.hash }}
key: macos15-arm-go-mod-${{ steps.go-mod-hash.outputs.hash }}
- name: Generate Conan Hash
id: conan-hash
run: |
@@ -57,5 +57,5 @@ runs:
uses: actions/cache/save@v4
with:
path: ~/.conan
key: macos15-conan-${{ steps.conan-hash.outputs.hash }}
key: macos15-arm-conan-${{ steps.conan-hash.outputs.hash }}
.github/workflows/code-checker.yaml
+5 -5
View File
@@ -101,8 +101,8 @@ jobs:
rockylinux:
if: ${{ false }} # skip for now
name: Code Checker rockylinux8
# Run in amazonlinux docker
name: Code Checker rockylinux9
# Run in rockylinux docker
runs-on: ubuntu-latest
timeout-minutes: 180
steps:
@@ -120,13 +120,13 @@ jobs:
- name: Download Caches
uses: ./.github/actions/cache-restore
with:
os: 'rockylinux8'
os: 'rockylinux9'
- name: Code Check
run: |
sed -i 's/ubuntu22.04/rockylinux8/g' .env
sed -i 's/ubuntu22.04/rockylinux9/g' .env
./build/builder.sh /bin/bash -c "make install"
- name: Save Caches
uses: ./.github/actions/cache-save
if: github.event_name != 'pull_request'
with:
os: 'rockylinux8'
os: 'rockylinux9'
.github/workflows/mac.yaml
+2 -2
View File
@@ -43,7 +43,7 @@ concurrency:
jobs:
mac:
name: Code Checker MacOS
runs-on: macos-15-intel
runs-on: macos-15
timeout-minutes: 480
steps:
- name: Checkout
@@ -107,7 +107,7 @@ jobs:
if [[ ! -d "${BREW_PREFIX}/opt/llvm" ]]; then
ln -s ${BREW_PREFIX}/opt/llvm@17 ${BREW_PREFIX}/opt/llvm
fi
pip3 install conan==1.64.1
pip3 install conan==1.66.0
make milvus
- name: Upload Cmake log
uses: actions/upload-artifact@v4
.github/workflows/publish-builder.yaml
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu20.04, ubuntu22.04, amazonlinux2023, rockylinux8]
os: [ubuntu24.04, ubuntu22.04, amazonlinux2023, rockylinux9]
env:
OS_NAME: ${{ matrix.os }}
IMAGE_ARCH: ${{ matrix.arch }}
DEVELOPMENT.md
+5 -5
View File
@@ -69,7 +69,7 @@ The following specification (either physical or virtual machine resources) is re
|----------|----------|--------------|-------|
| macOS | 12, 13, 14, 15 | Intel (x86_64), Apple Silicon (arm64) | Requires Homebrew |
| Ubuntu | 20.04, 22.04, 24.04 | x86_64, arm64 | |
| Rocky Linux | 8, 9 | x86_64, arm64 | RHEL compatible |
| Rocky Linux | 9 | x86_64, arm64 | RHEL compatible |
| Amazon Linux | 2023 | x86_64, arm64 | |
| CentOS | 7 (legacy) | x86_64 | EOL, use Rocky Linux instead |
@@ -79,8 +79,8 @@ The following specification (either physical or virtual machine resources) is re
| Platform | Compiler | Supported Versions |
|----------|----------|-------------------|
| macOS | LLVM/Clang | 14, 15, 16, 17, 18 |
| Linux | GCC | 9, 10, 11, 12, 13, 14 |
| macOS | LLVM/Clang | 15, 16, 17 |
| Linux | GCC | 11, 12, 13, 14 |
**Tool Requirements:**
@@ -172,7 +172,7 @@ Milvus uses Conan to manage third-party dependencies for c++.
Install Conan:
```shell
pip install conan==1.64.1
pip install conan==1.66.0
```
Note: Conan version 2.x is not currently supported, please use version 1.64.1.
@@ -480,7 +480,7 @@ A: This is a known issue on macOS 15+. Try using a newer version of LLVM:
brew install llvm@17
```
The build scripts will automatically detect and use LLVM 14-18.
The build scripts will automatically detect and use LLVM 15-17.
---
README.md
+1 -1
View File
@@ -169,7 +169,7 @@ Requirements:
```bash
Go: >= 1.21
CMake: >= 3.26.4 && CMake < 4
GCC: 9.5
GCC: >= 11
Python: > 3.8 and <= 3.11
```
README_CN.md
+1 -1
View File
@@ -72,7 +72,7 @@ Milvus 基于 [Apache 2.0 License](https://github.com/milvus-io/milvus/blob/mast
```
go: 1.21
cmake: >=3.18
gcc: 7.5
gcc: >= 11
protobuf: >=3.7
```
build/docker/builder/cpu/amazonlinux2023/Dockerfile
+2 -2
View File
@@ -15,7 +15,7 @@ ARG TARGETARCH
RUN dnf install -y wget g++ gcc gdb libatomic libstdc++-static ninja-build git make zip unzip tar which \
autoconf automake python3 python3-pip perl-FindBin texinfo \
pkg-config libuuid-devel libaio perl-IPC-Cmd libasan openblas-devel tzdata && \
pkg-config libuuid-devel libaio perl-IPC-Cmd perl-Digest-SHA libasan openblas-devel tzdata && \
rm -rf /var/cache/yum/*
ENV GOPATH /go
@@ -27,7 +27,7 @@ RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARG
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
RUN pip3 install conan==1.64.1
RUN pip3 install conan==1.66.0
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/cpu/rockylinux8/Dockerfile
-39
View File
@@ -1,39 +0,0 @@
FROM rockylinux/rockylinux:8 as vcpkg-installer
RUN dnf -y install curl wget tar zip unzip git \
gcc gcc-c++ make cmake \
perl-IPC-Cmd perl-Digest-SHA
# install ninjia
RUN dnf -y update && \
dnf -y install dnf-plugins-core && \
dnf config-manager --set-enabled powertools && \
dnf -y install ninja-build
########################################################################################
FROM rockylinux/rockylinux:8
ARG TARGETARCH
RUN dnf install -y make cmake automake gcc gcc-c++ curl zip unzip tar git which \
libaio libuuid-devel wget python3 python3-pip \
pkg-config perl-IPC-Cmd perl-Digest-SHA libatomic libtool tzdata
# install openblas-devel texinfo ninja
RUN dnf -y update && \
dnf -y install dnf-plugins-core && \
dnf config-manager --set-enabled powertools && \
dnf -y install texinfo openblas-devel ninja-build
RUN pip3 install conan==1.64.1
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain=1.89 -y
ENV PATH=/root/.cargo/bin:/usr/local/bin:/usr/local/go/bin:$PATH
COPY --chown=0:0 build/docker/builder/entrypoint.sh /
ENTRYPOINT [ "/entrypoint.sh" ]
CMD ["tail", "-f", "/dev/null"]
build/docker/builder/cpu/rockylinux9/Dockerfile
+55
View File
@@ -0,0 +1,55 @@
# Copyright (C) 2019-2022 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
FROM rockylinux/rockylinux:9
ARG TARGETARCH
# Rocky 9 ships with GCC 11 and Python 3.9 by default.
# Enable CRB (CodeReady Builder) repo for openblas-devel, texinfo, etc.
RUN dnf install -y dnf-plugins-core epel-release && \
dnf config-manager --set-enabled crb && \
dnf install -y \
wget g++ gcc gdb libatomic libstdc++-static ninja-build git make zip unzip tar which \
autoconf automake python3 python3-pip perl-FindBin texinfo ccache \
pkg-config libuuid-devel libaio libaio-devel perl-IPC-Cmd perl-Digest-SHA \
libasan openblas-devel tzdata libtool && \
rm -rf /var/cache/dnf/* && \
for f in /usr/include/openblas/*.h; do ln -sf "$f" /usr/include/$(basename "$f"); done
ENV GOPATH /go
ENV GOROOT /usr/local/go
ENV GO111MODULE on
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
RUN pip3 install conan==1.66.0
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
/home/milvus/.vscode-server-insiders/extensions \
&& chmod -R 777 /home/milvus
COPY --chown=0:0 build/docker/builder/entrypoint.sh /
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain=1.89 -y
ENV PATH=/root/.cargo/bin:$PATH
ENTRYPOINT [ "/entrypoint.sh" ]
CMD ["tail", "-f", "/dev/null"]
build/docker/builder/cpu/ubuntu20.04/Dockerfile
+11 -2
View File
@@ -16,14 +16,23 @@ ARG TARGETARCH
RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 \
g++ gcc gdb gdbserver ninja-build git make ccache libssl-dev zlib1g-dev zip unzip \
clang-format-12 clang-tidy-12 lcov libtool m4 autoconf automake python3 python3-pip \
pkg-config uuid-dev libaio-dev libopenblas-dev && \
pkg-config uuid-dev libaio-dev libopenblas-dev software-properties-common && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \
ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
echo "Etc/UTC" > /etc/timezone && \
apt-get remove --purge -y && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install conan==1.64.1
# Upgrade GCC to 11 (focal default is 9, which folly 2024.08.12.00 rejects)
RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
apt-get update && apt-get install -y gcc-11 g++-11 && \
cd /usr/bin && \
unlink gcc && ln -s gcc-11 gcc && \
unlink g++ && ln -s g++-11 g++ && \
unlink gcov && ln -s gcov-11 gcov && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install conan==1.66.0
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/cpu/ubuntu22.04/Dockerfile
+1 -1
View File
@@ -29,7 +29,7 @@ RUN apt-get update && apt-get install -y gcc-12 g++-12 && cd /usr/bin \
&& unlink g++ && ln -s g++-12 g++ \
&& unlink gcov && ln -s gcov-12 gcov
RUN pip3 install conan==1.64.1
RUN pip3 install conan==1.66.0
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.31/cmake-3.31.8-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/cpu/ubuntu24.04/Dockerfile
+56
View File
@@ -0,0 +1,56 @@
# Copyright (C) 2019-2022 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
FROM ubuntu:noble-20240530
ARG TARGETARCH
RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 \
g++ gcc gdb gdbserver ninja-build git make ccache libssl-dev zlib1g-dev zip unzip \
clang-format-18 clang-tidy-18 lcov libtool m4 autoconf automake python3 python3-pip pipx \
pkg-config uuid-dev libaio-dev libopenblas-dev && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \
ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
echo "Etc/UTC" > /etc/timezone && \
apt-get remove --purge -y && \
rm -rf /var/lib/apt/lists/*
# Ubuntu 24.04 ships GCC 13 by default, no upgrade needed
RUN pip3 install --break-system-packages conan==1.66.0
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.31/cmake-3.31.8-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
# Install Go
ENV GOPATH /go
ENV GOROOT /usr/local/go
ENV GO111MODULE on
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
/home/milvus/.vscode-server-insiders/extensions \
&& chmod -R 777 /home/milvus
COPY --chown=0:0 build/docker/builder/entrypoint.sh /
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain=1.89 -y
ENV PATH=/root/.cargo/bin:$PATH
ENTRYPOINT [ "/entrypoint.sh" ]
CMD ["tail", "-f", "/dev/null"]
build/docker/builder/entrypoint.sh
+4
View File
@@ -26,6 +26,10 @@ if [ -f "/etc/profile.d/devtoolset-11.sh" ]; then
source "/etc/profile.d/devtoolset-11.sh"
fi
if [ -f "/opt/rh/gcc-toolset-11/enable" ]; then
source "/opt/rh/gcc-toolset-11/enable"
fi
if [ -f "/etc/profile.d/llvm-toolset-11.sh" ]; then
source "/etc/profile.d/llvm-toolset-11.sh"
fi
build/docker/builder/gpu/ubuntu20.04/Dockerfile
+1 -1
View File
@@ -25,7 +25,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
apt-get remove --purge -y && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install conan==1.64.1
RUN pip3 install conan==1.66.0
# Install openblas
# RUN wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
build/docker/builder/gpu/ubuntu22.04/Dockerfile
+1 -1
View File
@@ -18,7 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
# Install go
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.24.12.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go
# Install conan
RUN pip3 install conan==1.64.1
RUN pip3 install conan==1.66.0
# Install rust
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain=1.89 -y
build/docker/milvus/amazonlinux2023/Dockerfile
+4
View File
@@ -35,9 +35,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
# Change user to milvus
# protobuf 5.x RepeatedField adds container annotations that trigger ASAN
# container-overflow false positives during static initialization, disable it.
ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0
RUN groupadd -r milvus && useradd -r -g milvus milvus && \
mkdir -p /var/lib/milvus && \
chown -R milvus:milvus /milvus /var/lib/milvus
ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0
USER milvus:milvus
ENTRYPOINT ["/tini", "--"]
build/docker/milvus/rockylinux8/Dockerfile → build/docker/milvus/rockylinux9/Dockerfile
+11 -8
View File
@@ -1,5 +1,4 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
@@ -10,17 +9,17 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
FROM rockylinux/rockylinux:8
FROM rockylinux/rockylinux:9
ARG TARGETARCH
ARG MILVUS_ASAN_LIB
RUN dnf install -y wget libgomp libaio libatomic tzdata
# install openblas-devel
RUN dnf -y install dnf-plugins-core && \
dnf config-manager --set-enabled powertools && \
dnf -y install openblas-devel
# Enable CRB repo for openblas-devel
RUN dnf install -y dnf-plugins-core && \
dnf config-manager --set-enabled crb && \
dnf install -y wget libgomp libaio libatomic openblas-devel tzdata && \
rm -rf /var/cache/dnf/* && \
for f in /usr/include/openblas/*.h; do ln -sf "$f" /usr/include/$(basename "$f"); done
# Add Tini
RUN curl -L -o /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini-$TARGETARCH && \
@@ -39,9 +38,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
# Change user to milvus
# protobuf 5.x RepeatedField adds container annotations that trigger ASAN
# container-overflow false positives during static initialization, disable it.
ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0
RUN groupadd -r milvus && useradd -r -g milvus milvus && \
mkdir -p /var/lib/milvus && \
chown -R milvus:milvus /milvus /var/lib/milvus
ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0
USER milvus:milvus
ENTRYPOINT ["/tini", "--"]
build/docker/milvus/ubuntu20.04/Dockerfile
+4
View File
@@ -43,9 +43,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
# Change user to milvus
# protobuf 5.x RepeatedField adds container annotations that trigger ASAN
# container-overflow false positives during static initialization, disable it.
ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0
RUN groupadd -r milvus && useradd -r -g milvus milvus && \
mkdir -p /var/lib/milvus && \
chown -R milvus:milvus /milvus /var/lib/milvus
ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0
USER milvus:milvus
ENTRYPOINT ["/tini", "--"]
build/docker/milvus/ubuntu22.04/Dockerfile
+4 -2
View File
@@ -43,11 +43,13 @@ ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
# Change user to milvus
ENV ASAN_OPTIONS=detect_leaks=0
# protobuf 5.x RepeatedField adds container annotations that trigger ASAN
# container-overflow false positives during static initialization, disable it.
ENV ASAN_OPTIONS=detect_leaks=0:detect_container_overflow=0
RUN groupadd -r milvus && useradd -r -g milvus milvus && \
mkdir -p /var/lib/milvus && \
chown -R milvus:milvus /milvus /var/lib/milvus
ENV ASAN_OPTIONS=detect_leaks=1
ENV ASAN_OPTIONS=detect_leaks=1:detect_container_overflow=0
USER milvus:milvus
ENTRYPOINT ["/tini", "--"]
ci/jenkins/PR-for-go-sdk.groovy
+7 -3
View File
@@ -11,7 +11,7 @@ pipeline {
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
preserveStashes(buildCount: 5)
disableConcurrentBuilds(abortPrevious: true)
timeout(time: 2, unit: 'HOURS')
timeout(time: 4, unit: 'HOURS')
throttleJobProperty(
categories: ['go-sdk'],
throttleEnabled: true,
@@ -58,7 +58,9 @@ pipeline {
pullRequestNumber: "$env.CHANGE_ID",
suppress_suffix_of_image_tag: true,
make_cmd: "make clean && make jobs=8 install use_disk_index=ON",
images: '["milvus","gotestsum","helm"]'
images: '["milvus","gotestsum","helm"]',
tekton_log_timeout: '30m',
tekton_pipeline_timeout: '3h'
milvus_image_tag = tekton.query_result job_name, 'milvus-image-tag'
milvus_sdk_go_image = tekton.query_result job_name, 'gotestsum-image-fqdn'
@@ -104,7 +106,9 @@ pipeline {
milvus_sdk_go_image: milvus_sdk_go_image,
helm_image: helm_image,
milvus_deployment_option: milvus_deployment_option,
verbose: 'false'
verbose: 'false',
tekton_log_timeout: '30m',
tekton_pipeline_timeout: '3h'
}
}
}
ci/jenkins/PR.groovy
+1 -1
View File
@@ -10,7 +10,7 @@ pipeline {
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
preserveStashes(buildCount: 5)
disableConcurrentBuilds(abortPrevious: true)
timeout(time: 3, unit: 'HOURS')
timeout(time: 4, unit: 'HOURS')
throttleJobProperty(
categories: ['cpu-e2e'],
throttleEnabled: true,
cmd/tools/migration/backend/backup_header.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: backup_header.proto
package backend
internal/core/CMakeLists.txt
+16
View File
@@ -97,6 +97,11 @@ set( MILVUS_THIRDPARTY_SRC ${PROJECT_SOURCE_DIR}/thirdparty )
# self-installed dynamic libraries will be correctly linked by excutable
set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" "${CMAKE_INSTALL_PREFIX}/lib64" )
set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
# FetchContent-built shared libs (libbson, libbsoncxx, etc.) are placed in
# ${CMAKE_BINARY_DIR}/lib. The linker needs -rpath-link to resolve transitive
# shared library dependencies (e.g. libbsoncxx.so -> libbson-1.0.so) when
# linking test binaries.
set( CMAKE_BUILD_RPATH "${CMAKE_BINARY_DIR}/lib" )
# **************************** Dependencies ****************************
list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/conan )
@@ -164,6 +169,17 @@ if ( APPLE )
# Disable fixup chains to avoid __DATA_CONST segment issues with third-party libraries
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no_fixup_chains")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-no_fixup_chains")
# Folly's Exception.cpp references __cxa_increment/decrement_exception_refcount
# from libc++abi, which lives in <llvm-root>/lib/c++/ and is not on the default
# linker search path. Derive the path from CMAKE_CXX_COMPILER.
get_filename_component(_CXX_BIN_DIR "${CMAKE_CXX_COMPILER}" DIRECTORY)
get_filename_component(_LLVM_ROOT "${_CXX_BIN_DIR}" DIRECTORY)
set(_LLVM_CXXABI_DIR "${_LLVM_ROOT}/lib/c++")
if(EXISTS "${_LLVM_CXXABI_DIR}/libc++abi.dylib")
link_directories("${_LLVM_CXXABI_DIR}")
link_libraries(c++abi)
endif()
endif ()
# **************************** Coding style check tools ****************************
internal/core/conanfile.py
+29 -33
View File
@@ -11,44 +11,47 @@ class MilvusConan(ConanFile):
"nlohmann_json/3.11.3#ffb9e9236619f1c883e36662f944345d",
"zstd/1.5.5#34e9debe03bf0964834a09dfbc31a5dd",
"lz4/1.9.4#c5afb86edd69ac0df30e3a9e192e43db",
"snappy/1.1.9#0519333fef284acd04806243de7d3070",
"snappy/1.2.1#dfb7f7e837525210762d74e96eb4a3fc",
"arrow/17.0.0@milvus/dev-2.6#7af258a853e20887f9969f713110aac8",
"openssl/3.1.2#02594c4c0a6e2b4feb3cd15119993597",
"openssl/3.3.2#9f9f130d58e7c13e76bb8a559f0a6a8b",
"googleapis/cci.20221108#65604e1b3b9a6b363044da625b201a2a",
"gtest/1.13.0#f9548be18a41ccc6367efcb8146e92be",
"benchmark/1.7.0#459f3bb1a64400a886ba43047576df3c",
"protobuf/3.21.4#fd372371d994b8585742ca42c12337f9",
"protobuf/5.27.0@milvus/dev#6fff8583e2fe32babef04a9097f1d581",
"yaml-cpp/0.7.0#9c87b3998de893cf2e5a08ad09a7a6e0",
"marisa/0.2.6#68446854f5a420672d21f21191f8e5af",
"zlib/1.2.13#df233e6bed99052f285331b9f54d9070",
"libcurl/7.86.0#bbc887fae3341b3cb776c601f814df05",
"glog/0.6.0#d22ebf9111fed68de86b0fa6bf6f9c3f",
"fmt/9.1.0#95259249fb7ef8c6b5674a40b00abba3",
"zlib/1.3.1#f52e03ae3d251dec704634230cd806a2",
"libcurl/8.10.1#a3113369c86086b0e84231844e7ed0a9",
"libevent/2.1.12#b6333a128075d75a3614bd8418bf2099",
"glog/0.7.1#83889ae482004d816ecac59dfef0d65a",
"fmt/11.0.2#5c7438ef4d5d69ab106a41e460ce11f3",
"gflags/2.2.2#b15c28c567c7ade7449cf994168a559f",
"double-conversion/3.2.1#640e35791a4bac95b0545e2f54b7aceb",
"double-conversion/3.3.0#33321c201741cc32b51169c6d2d05e60",
"libsodium/cci.20220430#7429a9e5351cc67bea3537229921714d",
"xsimd/9.0.1#ac9fd02a381698c4e08c5c4ca03b73e1",
"xz_utils/5.4.0#a6d90890193dc851fa0d470163271c7a",
"prometheus-cpp/1.1.0#ea9b101cb785943adb40ad82eda7856c",
"xz_utils/5.4.5#b885d1d79c9d30cff3803f7f551dbe66",
"prometheus-cpp/1.2.4#0918d66c13f97acb7809759f9de49b3f",
"re2/20230301#f8efaf45f98d0193cd0b2ea08b6b4060",
"folly/2023.10.30.08@milvus/dev#81d7729cd4013a1b708af3340a3b04d9",
"google-cloud-cpp/2.28.0#cf2bffe2264488b6c1153fae2a8db095",
"opentelemetry-cpp/1.8.1.1@milvus/2.4#7345034855d593047826b0c74d9a0ced",
"folly/2024.08.12.00@milvus/dev#e09fc71826ce6b4568441910665f0889",
"google-cloud-cpp/2.28.0@milvus/dev#25e69d743269d6c9ae5bf676af2174dc",
"opentelemetry-cpp/1.23.0@milvus/dev#bcd65b63b8db8447178ed93bbc94dcc0",
"librdkafka/1.9.1#e24dcbb0a1684dcf5a56d8d0692ceef3",
"abseil/20230125.3#dad7cc4c83bbd44c1f1cc9cc4d97ac88",
"abseil/20250127.0#e6c46096070cc3fd060e95a837cd3fb2",
"roaring/3.0.0#25a703f80eda0764a31ef939229e202d",
"grpc/1.50.1@milvus/dev#75103960d1cac300cf425ccfccceac08",
"grpc/1.67.1@milvus/dev#5aa62c51bced448b83d7db9e5b3a13c7",
"rapidjson/cci.20230929#624c0094d741e6a3749d2e44d834b96c",
"simde/0.8.2#5e1edfd5cba92f25d79bf6ef4616b972",
"xxhash/0.8.3#199e63ab9800302c232d030b27accec0",
"unordered_dense/4.4.0#6a855c992618cc4c63019109a2e47298",
"geos/3.12.0#0b177c90c25a8ca210578fb9e2899c37",
"icu/74.2#cd1937b9561b8950a2ae6311284c5813",
"libavrocpp/1.12.1@milvus/dev",
"libavrocpp/1.12.1.1@milvus/dev#a77043b1b435c3abef7b45710d05b300",
)
generators = ("cmake", "cmake_find_package")
default_options = {
"openssl:shared": True,
"openssl:shared": False,
"openssl:no_apps": True,
"libevent:shared": True,
"double-conversion:shared": True,
"folly:shared": True,
@@ -71,6 +74,7 @@ class MilvusConan(ConanFile):
"arrow:with_azure": True,
"arrow:with_s3": True,
"arrow:encryption": True,
"protobuf:shared": True,
"aws-sdk-cpp:config": True,
"aws-sdk-cpp:text-to-speech": False,
"aws-sdk-cpp:transfer": False,
@@ -81,39 +85,31 @@ class MilvusConan(ConanFile):
"glog:with_gflags": True,
"glog:shared": True,
"prometheus-cpp:with_pull": False,
"fmt:header_only": True,
"fmt:header_only": False,
"onetbb:tbbmalloc": False,
"onetbb:tbbproxy": False,
"gdal:shared": True,
"gflags:shared": True,
"gdal:shared": False,
"gdal:fPIC": True,
"icu:shared": False,
"icu:data_packaging": "library",
"xz_utils:shared": True,
"opentelemetry-cpp:with_stl": True,
}
def configure(self):
if self.settings.arch not in ("x86_64", "x86"):
del self.options["folly"].use_sse4_2
if self.settings.os == "Macos":
# By default abseil use static link but can not be compatible with macos X86
self.options["abseil"].shared = True
# abseil static linking on macOS (previously shared for X86 compat)
self.options["arrow"].with_jemalloc = False
# Disable Arrow's S3 support on macOS - the Conan aws-c-* packages
# (aws-c-cal, aws-c-io) use deprecated macOS Security framework APIs
# that were removed in macOS 15+. Instead, milvus-storage provides
# S3 support using Homebrew's aws-sdk-cpp (1.11.735+) which has
# the compatibility fixes.
self.options["arrow"].with_s3 = False
# Use OpenSSL for libcurl on macOS
self.options["libcurl"].with_ssl = "openssl"
def requirements(self):
if self.settings.os != "Macos":
self.requires("libunwind/1.7.2")
# On Linux, use Conan's aws-sdk-cpp (Arrow and milvus-storage both use it)
self.requires("aws-sdk-cpp/1.11.352@milvus/dev")
# On macOS, S3 support is provided by milvus-storage using Homebrew's
# aws-sdk-cpp (brew install aws-sdk-cpp). Arrow's S3 is disabled to avoid
# pulling in incompatible aws-c-* packages from Conan.
self.requires("libunwind/1.8.1#97965ef7da98cf1662e14219b14134f7")
self.requires("aws-sdk-cpp/1.11.692@milvus/dev#1e17deac19383217d291a01c23147b33")
def imports(self):
self.copy("*.dylib", "../lib", "lib")
internal/core/lsan_suppressions.txt
+12
View File
@@ -0,0 +1,12 @@
# LeakSanitizer suppressions for Milvus C++ unit tests
#
# These suppress known false positives caused by C++ runtime exception
# object caching in uninstrumented shared libraries (protobuf, abseil,
# grpc, etc.). The exception objects are properly caught and handled
# during execution, but their thread-local caches appear as leaks
# at program exit because LeakSanitizer cannot track allocations
# across shared library boundaries without frame pointers.
#
# See: https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer#suppressions
leak:__cxa_allocate_exception
internal/core/src/CMakeLists.txt
-11
View File
@@ -38,17 +38,6 @@ include_directories(
${MILVUS_STORAGE_INCLUDE_DIR}
)
if(APPLE)
# On macOS, aws-sdk-cpp comes from Homebrew instead of Conan.
# Add Homebrew include path AFTER Conan includes so Conan headers take precedence,
# avoiding conflicts (e.g., Boost version mismatch).
if(EXISTS "/opt/homebrew/include/aws")
include_directories("/opt/homebrew/include")
elseif(EXISTS "/usr/local/include/aws")
include_directories("/usr/local/include")
endif()
endif()
add_subdirectory( pb )
add_subdirectory( config )
add_subdirectory( common )
internal/core/src/common/VectorArrayTest.cpp
+7 -7
View File
@@ -29,8 +29,8 @@
using namespace milvus;
TEST(VectorArray, TestSchema) {
namespace pb = milvus::proto;
pb::schema::CollectionSchema proto;
namespace milvus_pb = milvus::proto;
milvus_pb::schema::CollectionSchema proto;
proto.set_name("col");
proto.set_description("asdfhsalkgfhsadg");
auto dim = 16;
@@ -48,7 +48,7 @@ TEST(VectorArray, TestSchema) {
field->set_fieldid(100);
field->set_is_primary_key(true);
field->set_description("asdgfsagf");
field->set_data_type(pb::schema::DataType::Int64);
field->set_data_type(milvus_pb::schema::DataType::Int64);
}
{
@@ -60,14 +60,14 @@ TEST(VectorArray, TestSchema) {
field->set_name("struct_key");
field->set_nullable(false);
field->set_fieldid(102);
field->set_data_type(pb::schema::DataType::Array);
field->set_element_type(pb::schema::DataType::Int64);
field->set_data_type(milvus_pb::schema::DataType::Array);
field->set_element_type(milvus_pb::schema::DataType::Int64);
auto field2 = struct_field->add_fields();
field2->set_name("struct_float_vec");
field2->set_fieldid(103);
field2->set_data_type(pb::schema::DataType::ArrayOfVector);
field2->set_element_type(pb::schema::DataType::FloatVector);
field2->set_data_type(milvus_pb::schema::DataType::ArrayOfVector);
field2->set_element_type(milvus_pb::schema::DataType::FloatVector);
auto param = field2->add_type_params();
param->set_key("dim");
param->set_value("16");
internal/core/src/common/logging_c.cpp
+1 -1
View File
@@ -125,6 +125,6 @@ InitGoogleLoggingWithoutZapSink() {
void
GoogleLoggingAtLevel(int severity, const char* msg) {
google::LogAtLevel(severity, msg);
google::LogAtLevel(static_cast<google::LogSeverity>(severity), msg);
}
}
internal/core/src/exec/expression/Expr.cpp
+1 -1
View File
@@ -98,7 +98,7 @@ ExprSet::Eval(int32_t begin,
// Create TTL field filtering expression if schema has TTL field configured
// Returns a single OR expression: ttl_field is null OR ttl_field > physical_us
// This means: keep entities with null TTL (never expire) OR entities with TTL > current time (not expired)
inline expr::TypedExprPtr
expr::TypedExprPtr
CreateTTLFieldFilterExpression(QueryContext* query_context) {
auto segment = query_context->get_segment();
auto& schema = segment->get_schema();
internal/core/src/exec/expression/GISFunctionFilterExpr.cpp
+4 -2
View File
@@ -345,7 +345,7 @@ PhyGISFunctionFilterExpr::EvalForDataSegment() {
default: {
ThrowInfo(NotImplemented,
"internal error: unknown GIS op : {}",
expr_->op_);
static_cast<int>(expr_->op_));
}
}
return res_vec;
@@ -455,7 +455,9 @@ PhyGISFunctionFilterExpr::EvalForIndexSegment() {
// Distance-based operation - no prepared version
return left.dwithin(query_geometry, expr_->distance_);
default:
ThrowInfo(NotImplemented, "unknown GIS op : {}", expr_->op_);
ThrowInfo(NotImplemented,
"unknown GIS op : {}",
static_cast<int>(expr_->op_));
}
};
internal/core/src/exec/operator/RescoresNode.cpp
+2 -1
View File
@@ -218,7 +218,8 @@ PhyRescoresNode::GetOutput() {
break;
default:
ThrowInfo(ErrorCode::UnexpectedError,
fmt::format("unknown boost boost mode: {}", boost_mode));
fmt::format("unknown boost boost mode: {}",
static_cast<int>(boost_mode)));
}
knowhere::MetricType metric_type = query_context_->get_metric_type();
internal/core/src/index/RTreeIndex.cpp
+8 -6
View File
@@ -15,6 +15,8 @@
#include <exception>
#include <list>
#include <map>
#include <mutex>
#include <shared_mutex>
#include <utility>
#include "boost/filesystem/directory.hpp"
@@ -139,7 +141,7 @@ RTreeIndex<T>::Load(milvus::tracer::TraceContext ctx, const Config& config) {
};
auto fill_null_offsets = [&](const uint8_t* data, int64_t size) {
folly::SharedMutexWritePriority::WriteHolder lock(mutex_);
std::unique_lock<folly::SharedMutexWritePriority> lock(mutex_);
null_offset_.resize((size_t)size / sizeof(size_t));
memcpy(null_offset_.data(), data, (size_t)size);
};
@@ -295,7 +297,7 @@ RTreeIndex<T>::BuildWithFieldData(
total_rows += n;
}
if (!local_nulls.empty()) {
folly::SharedMutexWritePriority::WriteHolder lock(mutex_);
std::unique_lock<folly::SharedMutexWritePriority> lock(mutex_);
null_offset_.reserve(null_offset_.size() + local_nulls.size());
null_offset_.insert(
null_offset_.end(), local_nulls.begin(), local_nulls.end());
@@ -373,7 +375,7 @@ RTreeIndex<T>::Upload(const Config& config) {
template <typename T>
BinarySet
RTreeIndex<T>::Serialize(const Config& config) {
folly::SharedMutexWritePriority::ReadHolder lock(mutex_);
std::shared_lock<folly::SharedMutexWritePriority> lock(mutex_);
auto bytes = null_offset_.size() * sizeof(size_t);
BinarySet res_set;
if (bytes > 0) {
@@ -412,7 +414,7 @@ const TargetBitmap
RTreeIndex<T>::IsNull() {
int64_t count = Count();
TargetBitmap bitset(count);
folly::SharedMutexWritePriority::ReadHolder lock(mutex_);
std::shared_lock<folly::SharedMutexWritePriority> lock(mutex_);
auto end = std::lower_bound(
null_offset_.begin(), null_offset_.end(), static_cast<size_t>(count));
for (auto it = null_offset_.begin(); it != end; ++it) {
@@ -426,7 +428,7 @@ TargetBitmap
RTreeIndex<T>::IsNotNull() {
int64_t count = Count();
TargetBitmap bitset(count, true);
folly::SharedMutexWritePriority::ReadHolder lock(mutex_);
std::shared_lock<folly::SharedMutexWritePriority> lock(mutex_);
auto end = std::lower_bound(
null_offset_.begin(), null_offset_.end(), static_cast<size_t>(count));
for (auto it = null_offset_.begin(); it != end; ++it) {
@@ -603,7 +605,7 @@ RTreeIndex<T>::AddGeometry(const std::string& wkb_data, int64_t row_offset) {
LOG_DEBUG("Added geometry at row offset {}", row_offset);
} else {
// Handle null geometry
folly::SharedMutexWritePriority::WriteHolder lock(mutex_);
std::unique_lock<folly::SharedMutexWritePriority> lock(mutex_);
null_offset_.push_back(static_cast<size_t>(row_offset));
// Update total row count
internal/core/src/index/VectorDiskIndex.cpp
+6 -2
View File
@@ -119,8 +119,10 @@ VectorDiskAnnIndex<T>::Load(milvus::tracer::TraceContext ctx,
{
auto read_file_span =
milvus::tracer::StartSpan("SegCoreReadDiskIndexFile", &ctx);
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>
read_file_nostd_span(read_file_span);
auto read_scope =
milvus::tracer::GetTracer()->WithActiveSpan(read_file_span);
opentelemetry::trace::Tracer::WithActiveSpan(read_file_nostd_span);
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
AssertInfo(index_files.has_value(),
@@ -139,8 +141,10 @@ VectorDiskAnnIndex<T>::Load(milvus::tracer::TraceContext ctx,
// start engine load index span
auto span_load_engine =
milvus::tracer::StartSpan("SegCoreEngineLoadDiskIndex", &ctx);
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>
nostd_span_load_engine(span_load_engine);
auto engine_scope =
milvus::tracer::GetTracer()->WithActiveSpan(span_load_engine);
opentelemetry::trace::Tracer::WithActiveSpan(nostd_span_load_engine);
auto stat = index_.Deserialize(knowhere::BinarySet(), load_config);
if (stat != knowhere::Status::success)
ThrowInfo(ErrorCode::UnexpectedError,
internal/core/src/index/VectorMemIndex.cpp
+6 -2
View File
@@ -264,8 +264,10 @@ VectorMemIndex<T>::Load(milvus::tracer::TraceContext ctx,
{
auto read_file_span =
milvus::tracer::StartSpan("SegCoreReadIndexFile", &ctx);
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>
read_file_nostd_span(read_file_span);
auto read_scope =
milvus::tracer::GetTracer()->WithActiveSpan(read_file_span);
opentelemetry::trace::Tracer::WithActiveSpan(read_file_nostd_span);
LOG_INFO("load with slice meta: {}", !slice_meta_filepath.empty());
auto load_priority =
@@ -348,8 +350,10 @@ VectorMemIndex<T>::Load(milvus::tracer::TraceContext ctx,
// start engine load index span
auto span_load_engine =
milvus::tracer::StartSpan("SegCoreEngineLoadIndex", &ctx);
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>
nostd_span_load_engine(span_load_engine);
auto engine_scope =
milvus::tracer::GetTracer()->WithActiveSpan(span_load_engine);
opentelemetry::trace::Tracer::WithActiveSpan(nostd_span_load_engine);
LOG_INFO("load index into Knowhere...");
LoadWithoutAssemble(binary_set, config);
span_load_engine->End();
internal/core/src/index/json_stats/JsonKeyStats.cpp
+1 -1
View File
@@ -1135,7 +1135,7 @@ JsonKeyStats::Load(milvus::tracer::TraceContext ctx, const Config& config) {
load_priority_ = config[milvus::LOAD_PRIORITY];
LOG_INFO("load json stats for segment {} with load priority: {}",
segment_id_,
load_priority_);
static_cast<int>(load_priority_));
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
internal/core/src/index/json_stats/utils.h
+3 -2
View File
@@ -593,7 +593,7 @@ template <>
struct fmt::formatter<milvus::index::JSONType> : fmt::formatter<std::string> {
template <typename FormatContext>
auto
format(const milvus::index::JSONType& jt, FormatContext& ctx) {
format(const milvus::index::JSONType& jt, FormatContext& ctx) const {
switch (jt) {
case milvus::index::JSONType::UNKNOWN:
return fmt::format_to(ctx.out(), "UNKNOWN");
@@ -631,7 +631,8 @@ template <>
struct fmt::formatter<milvus::index::JsonKeyLayoutType>
: fmt::formatter<std::string> {
auto
format(milvus::index::JsonKeyLayoutType type, fmt::format_context& ctx) {
format(milvus::index::JsonKeyLayoutType type,
fmt::format_context& ctx) const {
std::string name;
switch (type) {
case milvus::index::JsonKeyLayoutType::TYPED:
internal/core/src/mmap/ChunkVector.h
+3 -3
View File
@@ -100,7 +100,7 @@ class ThreadSafeChunkVector : public ChunkVectorBase<Type> {
AssertInfo(chunk_id < this->counter_,
fmt::format("index out of range, index={}, counter_={}",
chunk_id,
this->counter_));
this->counter_.load()));
if constexpr (!IsMmap || !IsVariableType<Type>) {
auto ptr = (Type*)vec_[chunk_id].data();
AssertInfo(
@@ -153,7 +153,7 @@ class ThreadSafeChunkVector : public ChunkVectorBase<Type> {
AssertInfo(index < this->counter_,
fmt::format("index out of range, index={}, counter_={}",
index,
this->counter_));
this->counter_.load()));
return vec_[index].data();
}
@@ -163,7 +163,7 @@ class ThreadSafeChunkVector : public ChunkVectorBase<Type> {
AssertInfo(index < this->counter_,
fmt::format("index out of range, index={}, counter_={}",
index,
this->counter_));
this->counter_.load()));
return vec_[index].size();
}
internal/core/src/mmap/ChunkVectorTest.cpp
-1
View File
@@ -22,7 +22,6 @@
using namespace milvus::segcore;
using namespace milvus;
namespace pb = milvus::proto;
class ChunkVectorTest : public ::testing::TestWithParam<bool> {
public:
void
internal/core/src/query/PlanProto.cpp
+1 -1
View File
@@ -112,7 +112,7 @@ ProtoParser::ParseSearchInfo(const planpb::VectorANNS& anns_proto) {
// check if hints is valid
ThrowInfo(ConfigInvalid,
"hints: {} not supported",
search_info.search_params_[HINTS]);
search_info.search_params_[HINTS].dump());
}
}
}
internal/core/src/rescores/Utils.h
+1 -1
View File
@@ -40,7 +40,7 @@ function_score_merge(const float& a,
ThrowInfo(ErrorCode::UnexpectedError,
fmt::format("unknown boost function mode: {}:{}",
proto::plan::FunctionMode_Name(mode),
mode));
static_cast<int>(mode)));
}
}
internal/core/src/segcore/SegmentGrowingIndexTest.cpp
-1
View File
@@ -57,7 +57,6 @@
using namespace milvus;
using namespace milvus::segcore;
namespace pb = milvus::proto;
using Param = std::tuple<DataType,
/*index type*/ std::string,
internal/core/src/segcore/SegmentGrowingStorageV2Test.cpp
-1
View File
@@ -60,7 +60,6 @@
using namespace milvus;
using namespace milvus::segcore;
namespace pb = milvus::proto;
class TestGrowingStorageV2 : public ::testing::Test {
void
internal/core/src/segcore/SegmentGrowingTest.cpp
-1
View File
@@ -63,7 +63,6 @@
using namespace milvus::segcore;
using namespace milvus;
namespace pb = milvus::proto;
TEST(Growing, DeleteCount) {
auto schema = std::make_shared<Schema>();
internal/core/src/storage/ChunkManager.cpp
+1 -13
View File
@@ -120,19 +120,7 @@ GcpChunkManager::GcpChunkManager(const StorageConfig& storage_config) {
remote_root_path_ = storage_config.root_path;
if (storage_config.useIAM) {
sdk_options_.httpOptions.httpClientFactory_create_fn = []() {
auto client_factory = [](google::cloud::Options const& opts)
-> std::unique_ptr<google::cloud::rest_internal::RestClient> {
return google::cloud::rest_internal::MakeDefaultRestClient(
{}, opts);
};
auto credentials = std::make_shared<
google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS::
ComputeEngineCredentials>(google::cloud::Options{},
client_factory);
return Aws::MakeShared<GoogleHttpClientFactory>(
GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials);
};
ConfigureGoogleCloudIAMHttpClientFactory(sdk_options_);
}
InitSDKAPIDefault(storage_config.log_level);
internal/core/src/storage/minio/MinioChunkManager.cpp
+18 -14
View File
@@ -49,6 +49,7 @@
#include "common/Consts.h"
#include "common/EasyAssert.h"
#include "fmt/core.h"
#include "google/cloud/internal/rest_client.h"
#include "google/cloud/storage/oauth2/compute_engine_credentials.h"
#include "google/cloud/version.h"
#include "log/Log.h"
@@ -64,6 +65,22 @@ namespace milvus::storage {
std::atomic<size_t> MinioChunkManager::init_count_(0);
std::mutex MinioChunkManager::client_mutex_;
void
ConfigureGoogleCloudIAMHttpClientFactory(Aws::SDKOptions& sdk_options) {
sdk_options.httpOptions.httpClientFactory_create_fn = []() {
auto credentials =
std::make_shared<google::cloud::oauth2_internal::
GOOGLE_CLOUD_CPP_NS::ComputeEngineCredentials>(
google::cloud::Options{},
[](google::cloud::Options const& opts) {
return google::cloud::rest_internal::MakeDefaultRestClient(
"", opts);
});
return Aws::MakeShared<GoogleHttpClientFactory>(
GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials);
};
}
static void
SwallowHandler(int signal) {
#pragma GCC diagnostic push
@@ -126,20 +143,7 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type,
sigaddset(&psa.sa_mask, SIGPIPE);
sigaction(SIGPIPE, &psa, 0);
if (type == RemoteStorageType::GOOGLE_CLOUD && useIAM) {
sdk_options_.httpOptions.httpClientFactory_create_fn = []() {
auto client_factory = [](google::cloud::Options const& opts)
-> std::unique_ptr<
google::cloud::rest_internal::RestClient> {
return google::cloud::rest_internal::MakeDefaultRestClient(
{}, opts);
};
auto credentials = std::make_shared<
google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS::
ComputeEngineCredentials>(google::cloud::Options{},
client_factory);
return Aws::MakeShared<GoogleHttpClientFactory>(
GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG, credentials);
};
ConfigureGoogleCloudIAMHttpClientFactory(sdk_options_);
}
LOG_INFO("init aws with log level:{}", log_level_str);
auto get_aws_log_level = [](const std::string& level_str) {
internal/core/src/storage/minio/MinioChunkManager.h
+3
View File
@@ -70,6 +70,9 @@ enum class RemoteStorageType {
ALIYUN_CLOUD = 2,
};
void
ConfigureGoogleCloudIAMHttpClientFactory(Aws::SDKOptions& sdk_options);
template <typename... Args>
static std::string
S3ErrorMessage(const std::string& func,
internal/core/thirdparty/knowhere/CMakeLists.txt
Vendored
+1 -47
View File
@@ -14,7 +14,7 @@
# Update KNOWHERE_VERSION for the first occurrence
milvus_add_pkg_config("knowhere")
set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
set( KNOWHERE_VERSION 3cade7e )
set( KNOWHERE_VERSION b20270cf507d6d3fb943a9c1d89a4093762a7b53 )
set( GIT_REPOSITORY "https://github.com/zilliztech/knowhere.git")
message(STATUS "Knowhere repo: ${GIT_REPOSITORY}")
@@ -52,52 +52,6 @@ FetchContent_GetProperties( knowhere )
if ( NOT knowhere_POPULATED )
FetchContent_Populate( knowhere )
# Patch knowhere cmake for macOS ARM64 - include hook.cc but exclude SVE sources
# SVE (Scalable Vector Extension) is not supported on Apple Silicon
# But hook.cc contains required symbols (fvec_hook, fvec_madd) needed for linking
if(APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
set(_KNOWHERE_FAISS_CMAKE "${knowhere_SOURCE_DIR}/cmake/libs/libfaiss.cmake")
file(READ "${_KNOWHERE_FAISS_CMAKE}" _FAISS_CMAKE_CONTENT)
# Verify patch targets exist (detect upstream changes early)
string(FIND "${_FAISS_CMAKE_CONTENT}" "set(UTILS_SRC src/simd/distances_ref.cc src/simd/distances_neon.cc)" _UTILS_POS)
if(_UTILS_POS EQUAL -1)
message(FATAL_ERROR "knowhere faiss patch failed: UTILS_SRC pattern not found in ${_KNOWHERE_FAISS_CMAKE}. "
"Upstream cmake may have changed (version: ${KNOWHERE_VERSION}).")
endif()
string(FIND "${_FAISS_CMAKE_CONTENT}" "set(UTILS_SVE_SRC src/simd/hook.cc src/simd/distances_sve.cc)" _SVE_POS)
if(_SVE_POS EQUAL -1)
message(FATAL_ERROR "knowhere faiss patch failed: UTILS_SVE_SRC pattern not found in ${_KNOWHERE_FAISS_CMAKE}. "
"Upstream cmake may have changed (version: ${KNOWHERE_VERSION}).")
endif()
string(FIND "${_FAISS_CMAKE_CONTENT}" "set(ALL_UTILS_SRC \${UTILS_SRC} \${UTILS_SVE_SRC})" _ALL_UTILS_POS)
if(_ALL_UTILS_POS EQUAL -1)
message(FATAL_ERROR "knowhere faiss patch failed: ALL_UTILS_SRC pattern not found in ${_KNOWHERE_FAISS_CMAKE}. "
"Upstream cmake may have changed (version: ${KNOWHERE_VERSION}).")
endif()
# Step 1: Add hook.cc to UTILS_SRC for Apple (contains required symbols)
string(REPLACE
"set(UTILS_SRC src/simd/distances_ref.cc src/simd/distances_neon.cc)"
"set(UTILS_SRC src/simd/distances_ref.cc src/simd/distances_neon.cc src/simd/hook.cc)"
_FAISS_CMAKE_CONTENT "${_FAISS_CMAKE_CONTENT}")
# Step 2: On Apple, exclude only distances_sve.cc from UTILS_SVE_SRC
string(REPLACE
"set(UTILS_SVE_SRC src/simd/hook.cc src/simd/distances_sve.cc)"
"set(UTILS_SVE_SRC src/simd/distances_sve.cc)"
_FAISS_CMAKE_CONTENT "${_FAISS_CMAKE_CONTENT}")
# Step 3: Exclude SVE sources on Apple
string(REPLACE
"set(ALL_UTILS_SRC \${UTILS_SRC} \${UTILS_SVE_SRC})"
"if(APPLE)\n set(ALL_UTILS_SRC \${UTILS_SRC})\n message(STATUS \"Excluding SVE sources on Apple (hook.cc included in UTILS_SRC)\")\n else()\n set(ALL_UTILS_SRC \${UTILS_SRC} \${UTILS_SVE_SRC})\n endif()"
_FAISS_CMAKE_CONTENT "${_FAISS_CMAKE_CONTENT}")
file(WRITE "${_KNOWHERE_FAISS_CMAKE}" "${_FAISS_CMAKE_CONTENT}")
message(STATUS "Patched knowhere cmake: hook.cc included, SVE excluded on Apple")
endif()
# Adding the following target:
# knowhere
add_subdirectory( ${knowhere_SOURCE_DIR}
internal/core/thirdparty/milvus-common/CMakeLists.txt
Vendored
+1 -2
View File
@@ -13,7 +13,7 @@
milvus_add_pkg_config("milvus-common")
set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
set( MILVUS-COMMON-VERSION f192a19 )
set( MILVUS-COMMON-VERSION e16257c98c8cbae7ca2d4b993838cdd95388fa62 )
set( GIT_REPOSITORY "https://github.com/zilliztech/milvus-common.git")
message(STATUS "milvus-common repo: ${GIT_REPOSITORY}")
@@ -32,7 +32,6 @@ FetchContent_Declare(
GIT_TAG ${MILVUS-COMMON-VERSION}
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/milvus-common-src
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/milvus-common-build
SOURCE_SUBDIR cpp
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} )
FetchContent_GetProperties( milvus-common )
internal/core/thirdparty/milvus-storage/CMakeLists.txt
Vendored
+41 -56
View File
@@ -28,7 +28,29 @@ else ()
set(WITH_AZURE_FS OFF CACHE BOOL "" FORCE )
endif ()
set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} )
# Disable benchmark for milvus-storage (we don't need it and it may have missing deps)
set(WITH_BENCHMARK OFF CACHE BOOL "" FORCE)
set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} ${CONAN_PROTOBUF_ROOT} )
# Make Conan's protoc visible to milvus-storage's find_package(Protobuf)
find_program(_PROTOC_PROGRAM protoc
PATHS ${CONAN_BIN_DIRS_PROTOBUF} ${CONAN_BIN_DIRS}
"${CMAKE_BINARY_DIR}/bin"
NO_DEFAULT_PATH)
if(NOT _PROTOC_PROGRAM)
file(GLOB _PROTOC_CANDIDATES
"${CONAN_BIN_DIRS_PROTOBUF}/protoc-*"
"${CMAKE_BINARY_DIR}/bin/protoc-*")
if(_PROTOC_CANDIDATES)
list(SORT _PROTOC_CANDIDATES)
list(GET _PROTOC_CANDIDATES -1 _PROTOC_PROGRAM)
endif()
endif()
if(_PROTOC_PROGRAM)
message(STATUS "Setting Protobuf_PROTOC_EXECUTABLE for milvus-storage: ${_PROTOC_PROGRAM}")
set(Protobuf_PROTOC_EXECUTABLE "${_PROTOC_PROGRAM}" CACHE FILEPATH "" FORCE)
endif()
FetchContent_Declare(
milvus-storage
GIT_REPOSITORY ${GIT_REPOSITORY}
@@ -42,60 +64,6 @@ FetchContent_GetProperties( milvus-storage )
if ( NOT milvus-storage_POPULATED )
FetchContent_Populate( milvus-storage )
if(APPLE)
# On macOS, Conan's aws-c-* packages use deprecated Security framework APIs
# removed in macOS 15+. Instead, use Homebrew's aws-sdk-cpp (1.11.735+)
# which has the compatibility fixes.
#
# Strategy: create an IMPORTED target with the Conan target name
# (AWS::aws-sdk-cpp-identity-management) that wraps Homebrew's AWS SDK.
# This way milvus-storage's unmodified CMakeLists.txt links correctly.
execute_process(
COMMAND brew --prefix
OUTPUT_VARIABLE _HOMEBREW_PREFIX
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
)
if(NOT _HOMEBREW_PREFIX)
set(_HOMEBREW_PREFIX "/opt/homebrew")
endif()
if(EXISTS "${_HOMEBREW_PREFIX}/lib/cmake/AWSSDK")
set(AWSSDK_DIR "${_HOMEBREW_PREFIX}/lib/cmake/AWSSDK")
else()
message(FATAL_ERROR "AWS SDK cmake config not found in Homebrew "
"(searched ${_HOMEBREW_PREFIX}/lib/cmake/AWSSDK). "
"Install with: brew install aws-sdk-cpp")
endif()
find_package(AWSSDK REQUIRED COMPONENTS core s3 identity-management)
# aws-sdk-cpp 1.11.750 had a packaging bug: the core CMakeLists.txt
# was missing the install() rule for pagination headers, so
# Paginator.h is absent from the installation even though the STS
# module's STSPaginationBase.h references it. Homebrew backported
# the fix (commit 175e803) with rebuild 1. Fail early with a clear
# message so developers know to reinstall.
if(NOT EXISTS "${_HOMEBREW_PREFIX}/include/aws/core/utils/pagination/Paginator.h")
message(FATAL_ERROR
"aws/core/utils/pagination/Paginator.h is missing from "
"Homebrew's aws-sdk-cpp installation. This is a known "
"packaging bug in aws-sdk-cpp 1.11.750. "
"Fix with: brew reinstall aws-sdk-cpp")
endif()
if(NOT TARGET AWS::aws-sdk-cpp-identity-management)
add_library(AWS::aws-sdk-cpp-identity-management INTERFACE IMPORTED)
set_target_properties(AWS::aws-sdk-cpp-identity-management PROPERTIES
INTERFACE_LINK_LIBRARIES "${AWSSDK_LINK_LIBRARIES}")
endif()
# Ignore Homebrew paths so milvus-storage's find_package(Boost) uses
# Conan's Boost 1.83, not Homebrew's Boost 1.90+ (ABI incompatible).
set(_SAVED_CMAKE_IGNORE_PATH "${CMAKE_IGNORE_PATH}")
list(APPEND CMAKE_IGNORE_PATH
"/opt/homebrew" "/opt/homebrew/include" "/opt/homebrew/lib"
"/usr/local" "/usr/local/include" "/usr/local/lib")
endif()
# Adding the following target:
# milvus-storage
add_subdirectory( ${milvus-storage_SOURCE_DIR}/cpp
@@ -104,8 +72,25 @@ if ( NOT milvus-storage_POPULATED )
# Enable S3 filesystem support on all platforms
target_compile_definitions(milvus-storage PRIVATE MILVUS_S3_FS)
# milvus-storage uses Aws::S3::S3Client and Aws::S3::Model::* directly but only
# links AWS::aws-sdk-cpp-identity-management in its own CMakeLists.txt. The S3
# symbols happen to resolve through Arrow's transitive dependency on aws-sdk-cpp-s3,
# but this is fragile (depends on Arrow package config). Explicitly link the S3
# component so the symbols are always resolved in libmilvus-storage.so.
# Conan's cmake_find_package generator creates FindAWSSDK.cmake but it must be
# explicitly loaded to create the AWS::* targets.
find_package(AWSSDK REQUIRED)
if(TARGET AWS::aws-sdk-cpp-s3)
target_link_libraries(milvus-storage PUBLIC AWS::aws-sdk-cpp-s3)
elseif(TARGET AWS::s3)
target_link_libraries(milvus-storage PUBLIC AWS::s3)
else()
message(WARNING "Neither AWS::aws-sdk-cpp-s3 nor AWS::s3 target found after find_package(AWSSDK)")
endif()
# On macOS, aws-c-io uses Apple's Network.framework for socket operations
if(APPLE)
set(CMAKE_IGNORE_PATH "${_SAVED_CMAKE_IGNORE_PATH}")
target_link_libraries(milvus-storage PUBLIC "-framework Network")
endif()
endif()
internal/core/unittest/CMakeLists.txt
+9 -2
View File
@@ -123,10 +123,11 @@ target_link_libraries(all_tests
knowhere
milvus-storage
milvus-common
milvus-planparser-cpp
)
# Add RPATH for plan parser library to find libmilvus-planparser.so at runtime
# Link plan parser library using full path and set RPATH
target_link_options(all_tests PRIVATE "-L${PLANPARSER_LIB_DIR}")
target_link_libraries(all_tests milvus-planparser-cpp)
set_target_properties(all_tests PROPERTIES
BUILD_RPATH "${PLANPARSER_LIB_DIR}"
INSTALL_RPATH "${PLANPARSER_LIB_DIR}"
@@ -161,4 +162,10 @@ target_link_libraries(bitset_test
gtest
${CONAN_LIBS}
)
if (LINUX)
# folly shared library requires libaio; use --no-as-needed to ensure
# the linker pulls it in even when bitset_test doesn't reference aio directly
target_link_options(bitset_test PRIVATE "LINKER:--no-as-needed")
target_link_libraries(bitset_test aio)
endif()
install(TARGETS bitset_test DESTINATION unittest)
internal/core/unittest/init_gtest.cpp
+21
View File
@@ -23,6 +23,13 @@
#include "exec/expression/function/init_c.h"
#include "folly/init/Init.h"
#include "milvus-storage/filesystem/fs.h"
#include "pb/cgo_msg.pb.h"
#include "pb/clustering.pb.h"
#include "pb/common.pb.h"
#include "pb/index_cgo_msg.pb.h"
#include "pb/plan.pb.h"
#include "pb/schema.pb.h"
#include "pb/segcore.pb.h"
#include "storage/LocalChunkManagerSingleton.h"
#include "storage/MmapManager.h"
#include "storage/RemoteChunkManagerSingleton.h"
@@ -38,6 +45,20 @@ main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
folly::Init follyInit(&argc, &argv, false);
// Force eager initialization of all protobuf descriptors.
// Protobuf 5.x uses lazy descriptor initialization via absl::call_once.
// Under ASAN, this lazy init can intermittently fail with
// "Check failed: file != nullptr" in AssignDescriptorsImpl.
// Initializing descriptors here (single-threaded, before any tests)
// eliminates the flaky failure.
::milvus::proto::common::SegmentStats::descriptor();
::milvus::proto::schema::CollectionSchema::descriptor();
::milvus::proto::plan::PlanNode::descriptor();
::milvus::proto::segcore::RetrieveResults::descriptor();
::milvus::proto::indexcgo::TypeParams::descriptor();
::milvus::proto::cgo::SerializedIndexFileInfo::descriptor();
::milvus::proto::clustering::StorageConfig::descriptor();
// Determine the base directory for test output.
// Priority: MILVUS_TEST_ROOT_DIR env var > compile-time MILVUS_CPPUT_OUTPUT_DIR
std::string base_dir;
internal/core/unittest/test_query.cpp
+12 -12
View File
@@ -475,8 +475,8 @@ TEST(Query, InnerProduct) {
}
TEST(Query, DISABLED_FillSegment) {
namespace pb = milvus::proto;
pb::schema::CollectionSchema proto;
namespace milvus_pb = milvus::proto;
milvus_pb::schema::CollectionSchema proto;
proto.set_name("col");
proto.set_description("asdfhsalkgfhsadg");
auto dim = 16;
@@ -494,7 +494,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_is_primary_key(false);
field->set_description("asdgfsagf");
field->set_fieldid(100);
field->set_data_type(pb::schema::DataType::FloatVector);
field->set_data_type(milvus_pb::schema::DataType::FloatVector);
auto param = field->add_type_params();
param->set_key("dim");
param->set_value("16");
@@ -510,7 +510,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(101);
field->set_is_primary_key(true);
field->set_description("asdgfsagf");
field->set_data_type(pb::schema::DataType::Int64);
field->set_data_type(milvus_pb::schema::DataType::Int64);
}
{
@@ -520,7 +520,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(102);
field->set_is_primary_key(false);
field->set_description("asdgfsagf");
field->set_data_type(pb::schema::DataType::Int32);
field->set_data_type(milvus_pb::schema::DataType::Int32);
}
auto schema = Schema::ParseFrom(proto);
@@ -556,7 +556,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(103);
field->set_is_primary_key(false);
field->set_description("lack null binlog");
field->set_data_type(pb::schema::DataType::Float);
field->set_data_type(milvus_pb::schema::DataType::Float);
}
{
@@ -566,7 +566,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(104);
field->set_is_primary_key(false);
field->set_description("lack default value binlog");
field->set_data_type(pb::schema::DataType::Bool);
field->set_data_type(milvus_pb::schema::DataType::Bool);
field->mutable_default_value()->set_bool_data(bool_default_value);
}
@@ -577,7 +577,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(105);
field->set_is_primary_key(false);
field->set_description("lack default value binlog");
field->set_data_type(pb::schema::DataType::Int32);
field->set_data_type(milvus_pb::schema::DataType::Int32);
field->mutable_default_value()->set_int_data(int_default_value);
}
@@ -588,7 +588,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(106);
field->set_is_primary_key(false);
field->set_description("lack default value binlog");
field->set_data_type(pb::schema::DataType::Int64);
field->set_data_type(milvus_pb::schema::DataType::Int64);
field->mutable_default_value()->set_int_data(long_default_value);
}
@@ -599,7 +599,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(107);
field->set_is_primary_key(false);
field->set_description("lack default value binlog");
field->set_data_type(pb::schema::DataType::Float);
field->set_data_type(milvus_pb::schema::DataType::Float);
field->mutable_default_value()->set_float_data(float_default_value);
}
@@ -610,7 +610,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(108);
field->set_is_primary_key(false);
field->set_description("lack default value binlog");
field->set_data_type(pb::schema::DataType::Double);
field->set_data_type(milvus_pb::schema::DataType::Double);
field->mutable_default_value()->set_double_data(double_default_value);
}
@@ -621,7 +621,7 @@ TEST(Query, DISABLED_FillSegment) {
field->set_fieldid(109);
field->set_is_primary_key(false);
field->set_description("lack default value binlog");
field->set_data_type(pb::schema::DataType::VarChar);
field->set_data_type(milvus_pb::schema::DataType::VarChar);
auto str_type_params = field->add_type_params();
str_type_params->set_key(MAX_LENGTH);
str_type_params->set_value(std::to_string(64));
pkg/eventlog/event_log.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: event_log.proto
package eventlog
pkg/eventlog/event_log_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: event_log.proto
package eventlog
pkg/proto/cgopb/cgo_msg.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: cgo_msg.proto
package cgopb
pkg/proto/clusteringpb/clustering.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: clustering.proto
package clusteringpb
pkg/proto/datapb/data_coord.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: data_coord.proto
package datapb
pkg/proto/datapb/data_coord_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: data_coord.proto
package datapb
pkg/proto/etcdpb/etcd_meta.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: etcd_meta.proto
package etcdpb
pkg/proto/indexcgopb/index_cgo_msg.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: index_cgo_msg.proto
package indexcgopb
pkg/proto/indexpb/index_coord.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: index_coord.proto
package indexpb
pkg/proto/indexpb/index_coord_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: index_coord.proto
package indexpb
pkg/proto/internalpb/internal.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: internal.proto
package internalpb
pkg/proto/messagespb/messages.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: messages.proto
package messagespb
pkg/proto/modelservicepb/model_service.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: model_service.proto
package modelservicepb
pkg/proto/modelservicepb/model_service_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: model_service.proto
package modelservicepb
pkg/proto/planpb/plan.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: plan.proto
package planpb
pkg/proto/proxypb/proxy.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: proxy.proto
package proxypb
pkg/proto/proxypb/proxy_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: proxy.proto
package proxypb
pkg/proto/querypb/query_coord.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: query_coord.proto
package querypb
pkg/proto/querypb/query_coord_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: query_coord.proto
package querypb
pkg/proto/rootcoordpb/root_coord.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: root_coord.proto
package rootcoordpb
pkg/proto/rootcoordpb/root_coord_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: root_coord.proto
package rootcoordpb
pkg/proto/segcorepb/segcore.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: segcore.proto
package segcorepb
pkg/proto/streamingpb/streaming.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: streaming.proto
package streamingpb
pkg/proto/streamingpb/streaming_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: streaming.proto
package streamingpb
pkg/proto/workerpb/worker.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v3.21.4
// protoc v5.27.0
// source: worker.proto
package workerpb
pkg/proto/workerpb/worker_grpc.pb.go
+1 -1
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.4
// - protoc v5.27.0
// source: worker.proto
package workerpb
scripts/3rdparty_build.sh
+131 -3
View File
@@ -88,11 +88,39 @@ if [[ -f "$HOME/.local/bin/conan" ]]; then
export PATH="$HOME/.local/bin:$PATH"
fi
# Ensure conan version matches the required version (1.66.0)
# CI Docker images may have an older version pre-installed
REQUIRED_CONAN_VERSION="1.66.0"
CURRENT_CONAN_VERSION=$(conan --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "0.0.0")
if [[ "${CURRENT_CONAN_VERSION}" != "${REQUIRED_CONAN_VERSION}" ]]; then
echo "Conan version mismatch: ${CURRENT_CONAN_VERSION} != ${REQUIRED_CONAN_VERSION}, upgrading..."
pip3 install conan==${REQUIRED_CONAN_VERSION} 2>/dev/null || pip install conan==${REQUIRED_CONAN_VERSION}
fi
pushd ${BUILD_OUTPUT_DIR}
# Unset LD_PRELOAD and LD_LIBRARY_PATH to prevent jemalloc and other shared
# libraries (set by setenv.sh) from being injected into conan's Python process.
# - LD_PRELOAD with jemalloc causes immediate segfaults in Python.
# - LD_LIBRARY_PATH with cmake_build/lib/ (populated by conan imports on
# previous runs) causes the Python ssl module to load a mismatched
# libssl.so during the conan update check, also triggering segfaults.
# The pre_build hook (below) handles setting LD_LIBRARY_PATH per-build
# for tools like grpc_cpp_plugin that need shared libs at build time.
SAVED_LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}"
unset LD_PRELOAD
unset LD_LIBRARY_PATH
export CONAN_REVISIONS_ENABLED=1
export CXXFLAGS="-Wno-error=address -Wno-error=deprecated-declarations -include cstdint"
export CFLAGS="-Wno-error=address -Wno-error=deprecated-declarations"
# LLVM from Homebrew doesn't set TARGET_OS_OSX=1 (unlike Apple's clang), which causes
# macOS SDK headers to exclude macOS-specific APIs (SecImportExport, SCPreferences, etc.).
# This fixes aws-c-cal, c-ares, and other packages that depend on macOS Security/SystemConfig APIs.
if [[ "$(uname -s)" == "Darwin" ]]; then
export CFLAGS="${CFLAGS} -DTARGET_OS_OSX=1"
export CXXFLAGS="${CXXFLAGS} -DTARGET_OS_OSX=1"
fi
# Allow CMake 4.x to build packages with old cmake_minimum_required versions (< 3.5)
export CMAKE_POLICY_VERSION_MINIMUM=3.5
@@ -103,6 +131,51 @@ if [[ ! `conan remote list` == *default-conan-local* ]]; then
conan remote add default-conan-local $CONAN_ARTIFACTORY_URL
fi
# Install a conan pre_build hook that sets LD_LIBRARY_PATH (Linux) or
# DYLD_LIBRARY_PATH (macOS) before each package build. This ensures that
# tools like grpc_cpp_plugin can find shared libraries (e.g. libprotoc.so)
# when invoked during downstream package builds (opentelemetry-cpp, googleapis).
# Use "conan config home" to get the correct conan home directory, as ~ may
# differ from the conan home in CI containers.
CONAN_HOME_DIR=$(conan config home 2>/dev/null || echo "${CONAN_USER_HOME:-$HOME}/.conan")
mkdir -p "$CONAN_HOME_DIR/hooks"
cat > "$CONAN_HOME_DIR/hooks/fix_shared_lib_env.py" << 'HOOK_EOF'
import os, platform
_saved_env = {}
def pre_build(output, conanfile, **kwargs):
"""Set library path before build so tools like grpc_cpp_plugin can find
shared libs (e.g. libprotoc.so). Saved state is restored in post_build
to avoid polluting the conan process environment."""
dep_lib_dirs = []
try:
for _, dep in conanfile.deps_cpp_info.dependencies:
for p in dep.lib_paths:
if os.path.isdir(p):
dep_lib_dirs.append(p)
except Exception:
return
if not dep_lib_dirs:
return
env_var = "DYLD_LIBRARY_PATH" if platform.system() == "Darwin" else "LD_LIBRARY_PATH"
_saved_env[env_var] = os.environ.get(env_var)
existing = os.environ.get(env_var, "")
new_val = ":".join(dep_lib_dirs) + (":" + existing if existing else "")
os.environ[env_var] = new_val
output.info("Set %s for %d dependency lib dirs" % (env_var, len(dep_lib_dirs)))
def post_build(output, conanfile, **kwargs):
"""Restore library path after build to prevent conan process corruption."""
for env_var, old_val in _saved_env.items():
if old_val is None:
os.environ.pop(env_var, None)
else:
os.environ[env_var] = old_val
_saved_env.clear()
HOOK_EOF
conan config set hooks.fix_shared_lib_env 2>/dev/null
unameOut="$(uname -s)"
case "${unameOut}" in
Darwin*)
@@ -111,7 +184,57 @@ case "${unameOut}" in
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
echo "Using CXX: $CXX"
echo "Using CC: $CC"
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; }
CONAN_ARGS="--install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -u"
# On macOS, Conan packages with shared libraries (protobuf, grpc) produce
# binaries (protoc, grpc_cpp_plugin) whose install rpaths don't include
# dependency lib dirs (e.g. abseil). This causes dyld failures when
# downstream packages (opentelemetry-cpp, googleapis) invoke them.
#
# Fix: install a Conan post_package hook that adds dependency rpaths to
# each package's binaries right after it is built (before the manifest is
# sealed), so every tool binary works the first time it is invoked.
mkdir -p "$CONAN_HOME_DIR/hooks"
conan config set hooks.fix_macos_rpaths 2>/dev/null
cat > "$CONAN_HOME_DIR/hooks/fix_macos_rpaths.py" << 'HOOK_EOF'
import os, platform, subprocess, stat
def post_package(output, conanfile, conanfile_path, **kwargs):
if platform.system() != "Darwin":
return
pkg = conanfile.package_folder
if not pkg:
return
bin_dir = os.path.join(pkg, "bin")
if not os.path.isdir(bin_dir):
return
dep_lib_dirs = []
try:
for _, dep in conanfile.deps_cpp_info.dependencies:
for p in dep.lib_paths:
if os.path.isdir(p):
dep_lib_dirs.append(p)
except Exception:
return
if not dep_lib_dirs:
return
fixed = []
for name in os.listdir(bin_dir):
fp = os.path.join(bin_dir, name)
if not os.path.isfile(fp) or not (os.stat(fp).st_mode & stat.S_IXUSR):
continue
r = subprocess.run(["file", fp], capture_output=True, text=True)
if "Mach-O" not in r.stdout:
continue
for lib_path in dep_lib_dirs:
subprocess.run(["install_name_tool", "-add_rpath", lib_path, fp],
capture_output=True)
fixed.append(name)
if fixed:
output.info("Fixed macOS rpaths for: %s" % ", ".join(fixed))
HOOK_EOF
conan install ${CPP_SRC_DIR} ${CONAN_ARGS} || { echo 'conan install failed'; exit 1; }
;;
Linux*)
if [ -f /etc/os-release ]; then
@@ -123,9 +246,9 @@ case "${unameOut}" in
export CPU_TARGET=avx
GCC_VERSION=`gcc -dumpversion`
if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -u || { echo 'conan install failed'; exit 1; }
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; }
else
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -u || { echo 'conan install failed'; exit 1; }
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; }
fi
;;
*)
@@ -133,6 +256,11 @@ case "${unameOut}" in
;;
esac
# Restore LD_LIBRARY_PATH so downstream build steps can find shared libs
if [[ -n "${SAVED_LD_LIBRARY_PATH}" ]]; then
export LD_LIBRARY_PATH="${SAVED_LD_LIBRARY_PATH}"
fi
popd
mkdir -p ${ROOT_DIR}/internal/core/output/lib
scripts/README.md
+1 -1
View File
@@ -6,7 +6,7 @@
OS: Ubuntu 20.04
go：1.21
cmake: >=3.18
gcc： 7.5
gcc: >= 11
```
## Install dependencies
scripts/check_proto_product.sh
+3 -1
View File
@@ -29,7 +29,9 @@ if [[ $(uname -s) == "Darwin" ]]; then
export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH"
fi
check_result=$(git status | grep -E ".*pb.go|.*pb.cc|.*pb.h")
# Use -I to ignore hunks where all changed lines are protoc version comments.
# This avoids false failures when CI protoc version differs from the committer's.
check_result=$(git diff -I '//.*protoc' --name-only -- '*.pb.go' '*.pb.cc' '*.pb.h')
echo "check_result: $check_result"
if test -z "$check_result"; then
exit 0
scripts/core_build.sh
+7 -9
View File
@@ -254,15 +254,13 @@ echo "CC $CC"
echo ${CMAKE_CMD}
${CMAKE_CMD} -G "${CMAKE_GENERATOR}"
# Export PROTOC for Rust prost-build (used by lance-encoding).
# Conan imports() copies the protoc binary into cmake_build/bin/.
# We prefer that over a system protoc to match the Conan protobuf version.
if [ -z "${PROTOC}" ] && [ -x "${BUILD_OUTPUT_DIR}/bin/protoc" ]; then
export PROTOC="${BUILD_OUTPUT_DIR}/bin/protoc"
echo "Using Conan protoc: ${PROTOC}"
elif [ -z "${PROTOC}" ] && command -v protoc &> /dev/null; then
export PROTOC="$(command -v protoc)"
echo "Using system protoc: ${PROTOC}"
# Export PROTOC for Rust crates (e.g. lance-encoding) that need it at build time
if [ -z "$PROTOC" ]; then
_PROTOC=$(grep -m1 "^Protobuf_PROTOC_EXECUTABLE" CMakeCache.txt 2>/dev/null | cut -d= -f2-)
if [ -n "$_PROTOC" ] && [ -f "$_PROTOC" ]; then
export PROTOC="$_PROTOC"
echo "Exported PROTOC=$PROTOC for Rust builds"
fi
fi
if [[ ${RUN_CPPLINT} == "ON" ]]; then
scripts/install_deps.sh
+22 -60
View File
@@ -21,12 +21,12 @@
# Supported platforms:
# - macOS 12, 13, 14, 15 (Intel and Apple Silicon)
# - Ubuntu 20.04, 22.04, 24.04
# - Rocky Linux 8, 9
# - Rocky Linux 9
# - Amazon Linux 2023
#
# Compiler requirements:
# - macOS: LLVM/Clang 14-18
# - Linux: GCC 9-14
# - macOS: LLVM/Clang 15-17
# - Linux: GCC 11-14
#
# Usage:
# ./scripts/install_deps.sh
@@ -57,7 +57,7 @@ print_error() {
# Minimum version requirements
MIN_CMAKE_VERSION="3.26"
MIN_GO_VERSION="1.21"
CONAN_VERSION="1.64.1"
CONAN_VERSION="1.66.0"
RUST_VERSION="1.89"
#######################################
@@ -198,11 +198,6 @@ install_mac_deps() {
if [[ "$arch" == "arm64" ]]; then
print_info "Installing Apple Silicon specific dependencies..."
brew install --quiet openssl librdkafka
# AWS SDK for S3 support on macOS (needed for milvus-storage)
brew install --quiet aws-sdk-cpp
else
# Intel Mac
brew install --quiet aws-sdk-cpp
fi
# Create symlink for LLVM (for scripts that expect /usr/local/opt/llvm)
@@ -328,60 +323,26 @@ install_rocky_deps() {
local rocky_version=$(detect_rocky_version)
print_info "Detected Rocky Linux ${rocky_version}"
# Enable EPEL repository
sudo dnf install -y epel-release
if [ "$rocky_version" -lt 9 ]; then
print_error "Rocky Linux ${rocky_version} is no longer supported. Please upgrade to Rocky Linux 9+."
exit 1
fi
# Base packages
local base_packages=(
wget curl which git make ninja-build
automake python3-devel python3-pip
openblas-devel libaio libuuid-devel
# Enable EPEL and CRB repositories
sudo dnf install -y epel-release dnf-plugins-core
sudo dnf config-manager --set-enabled crb
# Rocky 9 ships with GCC 11 by default — no toolset needed
print_info "Installing GCC and build dependencies for Rocky ${rocky_version}..."
sudo dnf install -y \
gcc gcc-c++ gcc-gfortran \
wget curl which git make ninja-build \
automake python3-devel python3-pip \
openblas-devel libaio libuuid-devel \
zip unzip ccache lcov libtool m4 autoconf
)
# Version-specific packages
case "$rocky_version" in
8)
print_info "Installing GCC toolset for Rocky 8..."
sudo dnf install -y gcc-toolset-12 gcc-toolset-12-libatomic-devel gcc-toolset-12-libstdc++-devel
sudo dnf install -y "${base_packages[@]}"
sudo dnf install -y atlas-devel
# Enable GCC toolset
echo "source /opt/rh/gcc-toolset-12/enable" | sudo tee /etc/profile.d/gcc-toolset-12.sh
source /opt/rh/gcc-toolset-12/enable
# Create libstdc++.a symlink for static linking (needed by conan packages like ninja)
local gcc12_libstdcxx="/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/libstdc++.a"
local system_gcc_dir="/usr/lib/gcc/x86_64-redhat-linux/8"
if [ -f "$gcc12_libstdcxx" ] && [ -d "$system_gcc_dir" ] && [ ! -f "$system_gcc_dir/libstdc++.a" ]; then
print_info "Creating libstdc++.a symlink for static linking..."
sudo ln -s "$gcc12_libstdcxx" "$system_gcc_dir/libstdc++.a"
fi
# Install LLVM toolset for clang-format
sudo dnf install -y llvm-toolset
echo "export CLANG_TOOLS_PATH=/usr/bin" | sudo tee -a /etc/profile.d/llvm-toolset.sh
;;
9)
print_info "Installing GCC for Rocky 9..."
sudo dnf install -y gcc gcc-c++ gcc-gfortran
sudo dnf install -y "${base_packages[@]}"
# Optionally install newer GCC toolset
sudo dnf install -y gcc-toolset-13 || true
if [ -d /opt/rh/gcc-toolset-13 ]; then
echo "source /opt/rh/gcc-toolset-13/enable" | sudo tee /etc/profile.d/gcc-toolset-13.sh
fi
# Install clang-tools
sudo dnf install -y clang-tools-extra || sudo dnf install -y clang
;;
*)
print_warn "Rocky Linux ${rocky_version} not explicitly supported"
sudo dnf install -y gcc gcc-c++ gcc-gfortran "${base_packages[@]}"
;;
esac
# Install clang-tools for formatting
sudo dnf install -y clang-tools-extra || sudo dnf install -y clang
# Install CMake if needed
install_cmake_linux
@@ -397,6 +358,7 @@ install_rocky_deps() {
conan profile new default --detect --force 2>/dev/null || true
conan profile update settings.compiler.version="$gcc_ver" default
conan profile update settings.compiler.libcxx=libstdc++11 default
conan profile update settings.compiler.cppstd=17 default
fi
# Install Rust
scripts/install_deps_msys.sh
+1 -1
View File
@@ -23,7 +23,7 @@ pacmanInstall()
mingw-w64-x86_64-diffutils \
mingw-w64-x86_64-go
pip3 install conan==1.64.1
pip3 install conan==1.66.0
}
updateKey()
scripts/run_cpp_codecov.sh
+10
View File
@@ -31,6 +31,12 @@ source ${ROOT_DIR}/scripts/setenv.sh
MILVUS_CORE_DIR="${ROOT_DIR}/internal/core"
MILVUS_CORE_UNITTEST_DIR="${MILVUS_CORE_DIR}/output/unittest"
# Suppress known LeakSanitizer false positives from uninstrumented shared libraries
LSAN_SUPPRESSIONS="${MILVUS_CORE_DIR}/lsan_suppressions.txt"
if [ -f "${LSAN_SUPPRESSIONS}" ]; then
export LSAN_OPTIONS="suppressions=${LSAN_SUPPRESSIONS}"
fi
echo "ROOT_DIR = ${ROOT_DIR}"
echo "MILVUS_CORE_DIR = ${MILVUS_CORE_DIR}"
echo "MILVUS_CORE_UNITTEST_DIR = ${MILVUS_CORE_UNITTEST_DIR}"
@@ -67,6 +73,10 @@ for test in $(ls ${MILVUS_CORE_UNITTEST_DIR}); do
# run unittest
if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
echo "ASAN is enabled with env MILVUS_ENABLE_ASAN_LIB, set {$MILVUS_ENABLE_ASAN_LIB} at the front of LD_PRELOAD"
# protobuf 5.x RepeatedField triggers ASAN container-overflow false positives during static init
# LSAN suppression: __cxa_allocate_exception false positives from uninstrumented shared libs
ASAN_OPTIONS="${ASAN_OPTIONS:+${ASAN_OPTIONS}:}detect_container_overflow=0" \
LSAN_OPTIONS="${LSAN_OPTIONS:+${LSAN_OPTIONS}:}suppressions=${MILVUS_CORE_DIR}/lsan_suppressions.txt" \
LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test}
else
${MILVUS_CORE_UNITTEST_DIR}/${test}
scripts/run_cpp_unittest.sh
+6
View File
@@ -35,6 +35,12 @@ if [ -d "${CORE_INSTALL_PREFIX}/lib" ]; then
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${CORE_INSTALL_PREFIX}/lib
fi
# Suppress known LeakSanitizer false positives from uninstrumented shared libraries
LSAN_SUPPRESSIONS="${MILVUS_CORE_DIR}/lsan_suppressions.txt"
if [ -f "${LSAN_SUPPRESSIONS}" ]; then
export LSAN_OPTIONS="suppressions=${LSAN_SUPPRESSIONS}"
fi
# run unittest
arg="$1"
filter_value="${arg#*=}"
scripts/run_cpp_ut.sh
+11 -1
View File
@@ -47,6 +47,12 @@ source ${ROOT_DIR}/scripts/setenv.sh
MILVUS_CORE_DIR="${ROOT_DIR}/internal/core"
MILVUS_CORE_UNITTEST_DIR="${MILVUS_CORE_DIR}/output/unittest"
# Suppress known LeakSanitizer false positives from uninstrumented shared libraries
LSAN_SUPPRESSIONS="${MILVUS_CORE_DIR}/lsan_suppressions.txt"
if [ -f "${LSAN_SUPPRESSIONS}" ]; then
export LSAN_OPTIONS="suppressions=${LSAN_SUPPRESSIONS}"
fi
echo "=============================================="
echo "=== C++ Unit Tests (No Coverage) ==="
echo "=============================================="
@@ -85,7 +91,11 @@ run_test() {
if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
echo "ASAN is enabled with env MILVUS_ENABLE_ASAN_LIB"
env $env_vars LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test}
# protobuf 5.x RepeatedField triggers ASAN container-overflow false positives during static init
# LSAN suppression: __cxa_allocate_exception false positives from uninstrumented shared libs
env $env_vars ASAN_OPTIONS="${ASAN_OPTIONS:+${ASAN_OPTIONS}:}detect_container_overflow=0" \
LSAN_OPTIONS="${LSAN_OPTIONS:+${LSAN_OPTIONS}:}suppressions=${MILVUS_CORE_DIR}/lsan_suppressions.txt" \
LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test}
else
env $env_vars ${MILVUS_CORE_UNITTEST_DIR}/${test}
fi
scripts/run_go_codecov.sh
+1
View File
@@ -56,6 +56,7 @@ for d in cmd/tools internal pkg client; do
-coverprofile="$PROFILE" \
-covermode=atomic \
-p "$PARALLEL" \
-ldflags="-r ${RPATH}" \
./...
if [ -f "$PROFILE" ]; then
scripts/run_intergration_test.sh
+2 -1
View File
@@ -45,8 +45,9 @@ TEST_CMD_WITH_ARGS=(
-coverpkg=./...
-coverprofile=profile.out
-covermode=atomic
-caseTimeout=20m
-caseTimeout=20m
-timeout=60m
-ldflags="-r ${RPATH}"
)
function test_cmd() {
scripts/setenv.sh
+37 -11
View File
@@ -51,20 +51,26 @@ case "${unameOut}" in
echo "WARN: Cannot find $LIBJEMALLOC"
fi
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:+$PKG_CONFIG_PATH:}$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64"
export RPATH=$LD_LIBRARY_PATH;;
MILVUS_LIB_DIRS="$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64:$ROOT_DIR/cmake_build/lib"
# LIBRARY_PATH: build-time linker search path (gcc/ld)
export LIBRARY_PATH="${LIBRARY_PATH:+$LIBRARY_PATH:}$MILVUS_LIB_DIRS"
# LD_LIBRARY_PATH: runtime linker search path - needed for transitive
# shared library dependencies (e.g. libfolly_exception_tracer.so has no
# RPATH and depends on libfolly_exception_tracer_base.so)
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$MILVUS_LIB_DIRS"
export RPATH=$MILVUS_LIB_DIRS;;
Darwin*)
# detect llvm version by valid list (supports LLVM 14-17)
# detect llvm version by valid list (supports LLVM 15-17)
# Note: LLVM 18 is NOT supported because Conan 1.x cannot handle the newer
# compiler profiles/settings that LLVM 18 requires. Until we migrate to Conan 2,
# please use LLVM 17 or earlier.
for llvm_version in 17 16 15 14 NOT_FOUND ; do
# please use LLVM 15-17.
for llvm_version in 17 16 15 NOT_FOUND ; do
if brew ls --versions llvm@${llvm_version} > /dev/null 2>&1; then
break
fi
done
if [ "${llvm_version}" = "NOT_FOUND" ] ; then
echo "ERROR: Valid LLVM (14-17) not installed. Run: brew install llvm@17"
echo "ERROR: Valid LLVM (15-17) not installed. Run: brew install llvm@17"
echo "NOTE: LLVM 18 is not supported due to Conan 1.x incompatibility."
exit 1
fi
@@ -74,14 +80,35 @@ case "${unameOut}" in
export CXX=${llvm_prefix}/bin/clang++
export ASM=${llvm_prefix}/bin/clang
macos_sdk_path="$(xcrun --show-sdk-path)"
export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include -isysroot ${macos_sdk_path}"
# LLVM from Homebrew doesn't set TARGET_OS_OSX=1 (unlike Apple's clang), causing
# macOS SDK headers to exclude macOS-specific APIs. Define it explicitly.
export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include -isysroot ${macos_sdk_path} -DTARGET_OS_OSX=1"
export CXXFLAGS=${CFLAGS}
export LDFLAGS="-L$(brew --prefix libomp)/lib"
# Include LLVM's own libc++/libc++abi so Conan packages that reference
# Apple-extension symbols (e.g. ___cxa_decrement_exception_refcount in
# folly) are resolved against the same runtime they were built with.
# -lc++abi is explicit because Homebrew LLVM clang++ does not inject it
# automatically (unlike Apple clang), but folly's static archive references it.
export LDFLAGS="-L$(brew --prefix libomp)/lib -L${llvm_prefix}/lib/c++ -lc++abi"
# Rust cc-rs crate uses the Xcode SDK version as deployment target if
# MACOSX_DEPLOYMENT_TARGET is unset. When the SDK version (e.g. 26.2) is
# newer than what this LLVM version understands, compilation fails.
# Cap it at the current macOS major version to stay compatible.
sdk_ver="$(xcrun --show-sdk-version 2>/dev/null)"
os_major="$(sw_vers -productVersion | cut -d. -f1)"
if [ -n "${sdk_ver}" ] && [ -n "${os_major}" ] && \
[ "$(echo "${sdk_ver}" | cut -d. -f1)" -gt "${os_major}" ]; then
export MACOSX_DEPLOYMENT_TARGET="${os_major}.0"
fi
# Rust cc-rs needs TARGET_OS_OSX for correct platform detection in aws-lc-sys.
# Set for both ARM and Intel targets so cross-compilation also works.
export CFLAGS_aarch64_apple_darwin="-DTARGET_OS_OSX=1"
export CFLAGS_x86_64_apple_darwin="-DTARGET_OS_OSX=1"
export CGO_CFLAGS="${CFLAGS}"
export CGO_LDFLAGS="${LDFLAGS} -framework Security -framework CoreFoundation"
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH:+$PKG_CONFIG_PATH:}$ROOT_DIR/internal/core/output/lib/pkgconfig"
export DYLD_LIBRARY_PATH=$ROOT_DIR/internal/core/output/lib
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig"
export DYLD_LIBRARY_PATH=$ROOT_DIR/cmake_build/lib:$ROOT_DIR/internal/core/output/lib
export RPATH=$DYLD_LIBRARY_PATH;;
MINGW*)
extra_path=$(cygpath -w "$ROOT_DIR/internal/core/output/lib")
@@ -91,4 +118,3 @@ case "${unameOut}" in
*)
echo "does not supported"
esac