furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-20 09:45:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Migrate to Conan 2.x (#48165)

Browse Source 
new JFrog artifactory for conan 2.x:
https://milvus01.jfrog.io/ui/repos/tree/General/default-conan-local2

issue: https://github.com/milvus-io/milvus/issues/47425

conanfiles pr: https://github.com/milvus-io/conanfiles/pull/92
milvus-common: https://github.com/zilliztech/milvus-common/pull/69
milvus-storage: https://github.com/milvus-io/milvus-storage/pull/445
knowhere: https://github.com/zilliztech/knowhere/pull/1495

Signed-off-by: yhmo <yihua.mo@zilliz.com>
This commit is contained in:
groot
2026-04-16 18:31:42 +08:00
committed by GitHub
parent 7d0b8df0c3
commit 499619033f
55 changed files with 351 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/cache-restore/action.yaml
+3 -3
View File
@@ -30,9 +30,9 @@ runs:
if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }}
uses: actions/cache/restore@v4
with:
path: .docker/amd64-${{ inputs.os }}-conan
key: ${{ inputs.os }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: ${{ inputs.os }}-conan-
path: .docker/amd64-${{ inputs.os }}-conan2
key: ${{ inputs.os }}-conan2-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: ${{ inputs.os }}-conan2-
- name: Cache Third Party
if: ${{ inputs.kind == 'all' || inputs.kind == 'thirdparty' }}
uses: actions/cache/restore@v4
.github/actions/cache-save/action.yaml
+3 -3
View File
@@ -23,9 +23,9 @@ runs:
if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }}
uses: actions/cache/save@v4
with:
path: .docker/amd64-${{ inputs.os }}-conan
key: ${{ inputs.os }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: ${{ inputs.os }}-conan-
path: .docker/amd64-${{ inputs.os }}-conan2
key: ${{ inputs.os }}-conan2-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: ${{ inputs.os }}-conan2-
- name: Cache Third Party
if: ${{ inputs.kind == 'all' || inputs.kind == 'thirdparty' }}
uses: actions/cache/save@v4
.github/actions/cache/action.yaml
+3 -3
View File
@@ -30,9 +30,9 @@ runs:
if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }}
uses: actions/cache@v3
with:
path: .docker/amd64-${{ inputs.os }}-conan
key: ${{ inputs.os }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: ${{ inputs.os }}-conan-
path: .docker/amd64-${{ inputs.os }}-conan2
key: ${{ inputs.os }}-conan2-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: ${{ inputs.os }}-conan2-
- name: Cache Third Party
if: ${{ inputs.kind == 'all' || inputs.kind == 'thirdparty' }}
uses: actions/cache@v3
.github/actions/macos-cache-restore/action.yaml
+3 -3
View File
@@ -49,6 +49,6 @@ runs:
- name: Restore Conan Cache
uses: actions/cache/restore@v4
with:
path: ~/.conan
key: macos15-arm-conan-${{ steps.conan-cache.outputs.hash }}
restore-keys: macos15-arm-conan-
path: ~/.conan2
key: macos15-arm-conan2-${{ steps.conan-cache.outputs.hash }}
restore-keys: macos15-arm-conan2-
.github/actions/macos-cache-save/action.yaml
+2 -2
View File
@@ -56,6 +56,6 @@ runs:
- name: Mac Cache Conan Packages
uses: actions/cache/save@v4
with:
path: ~/.conan
key: macos15-arm-conan-${{ steps.conan-hash.outputs.hash }}
path: ~/.conan2
key: macos15-arm-conan2-${{ steps.conan-hash.outputs.hash }}
.github/workflows/mac.yaml
+1 -1
View File
@@ -107,7 +107,7 @@ jobs:
if [[ ! -d "${BREW_PREFIX}/opt/llvm" ]]; then
ln -s ${BREW_PREFIX}/opt/llvm@17 ${BREW_PREFIX}/opt/llvm
fi
pip3 install conan==1.66.0
pip3 install conan==2.25.1
make milvus
- name: Upload Cmake log
uses: actions/upload-artifact@v4
.github/workflows/main.yaml
+1 -1
View File
@@ -96,7 +96,7 @@ jobs:
kind: 'cpp'
- name: Build
run: |
./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} mode=RelWithDebInfo build-cpp && chmod -R a+rwx /home/milvus/.conan"
./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} mode=RelWithDebInfo build-cpp && chmod -R a+rwx /home/milvus/.conan2"
- run: |
zip -r code.zip . -x "./.docker/*" -x "./cmake_build/thirdparty/**" -x ".git/**"
- name: Archive code
.gitignore
+1
View File
@@ -15,6 +15,7 @@
**/CMakeFiles/*
**/.cmake/*
CMakeCache.txt
CMakeUserPresets.json
.cache
coverage_report/
DEVELOPMENT.md
+3 -5
View File
@@ -88,7 +88,7 @@ The following specification (either physical or virtual machine resources) is re
|------|----------------|-------|
| Go | 1.21 | Required. Install from https://go.dev/dl/ |
| CMake | 3.26 | Auto-installed by install_deps.sh |
| Conan | 1.64.1 | Auto-installed. Conan 2.x not supported |
| Conan | 2.25.1 | Auto-installed by install_deps.sh |
| Rust | 1.89 | Auto-installed by install_deps.sh |
### Compiler Setup
@@ -172,11 +172,9 @@ Milvus uses Conan to manage third-party dependencies for c++.
Install Conan:
```shell
pip install conan==1.66.0
pip install conan==2.25.1
```
Note: Conan version 2.x is not currently supported, please use version 1.64.1.
#### Go
Milvus is written in [Go](http://golang.org/). If you don't have a Go development environment, please follow the instructions in the [Go Getting Started guide](https://golang.org/doc/install).
@@ -458,7 +456,7 @@ A: Python 3.12 has removed the imp module, please downgrade to 3.11 for now.
Q: Conan: Unrecognized arguments: — install-folder conan
A: The version is not correct. Please use Conan 1.64.1.
A: The version is not correct. Please use Conan 2.25.1.
---
build/builder.sh
+1 -1
View File
@@ -36,7 +36,7 @@ export IMAGE_ARCH=${PLATFORM_ARCH}
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-ccache"
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-go-mod"
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-vscode-extensions"
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan"
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan2"
chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker}"
$DOCKER_COMPOSE_COMMAND pull builder
build/docker/builder/cpu/amazonlinux2023/Dockerfile
+1 -1
View File
@@ -27,7 +27,7 @@ RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.25.8.linux-$TARGE
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
RUN pip3 install conan==1.66.0
RUN pip3 install conan==2.25.1
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/cpu/rockylinux9/Dockerfile
+1 -1
View File
@@ -34,7 +34,7 @@ RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.25.8.linux-$TARGE
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
RUN pip3 install conan==1.66.0
RUN pip3 install conan==2.25.1
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/cpu/ubuntu20.04/Dockerfile
+1 -1
View File
@@ -32,7 +32,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
unlink gcov && ln -s gcov-11 gcov && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install conan==1.66.0
RUN pip3 install conan==2.25.1
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.27/cmake-3.27.5-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/cpu/ubuntu22.04/Dockerfile
+1 -1
View File
@@ -29,7 +29,7 @@ RUN apt-get update && apt-get install -y gcc-12 g++-12 && cd /usr/bin \
&& unlink g++ && ln -s g++-12 g++ \
&& unlink gcov && ln -s gcov-12 gcov
RUN pip3 install conan==1.66.0
RUN pip3 install conan==2.25.1
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.31/cmake-3.31.8-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/cpu/ubuntu24.04/Dockerfile
+1 -1
View File
@@ -25,7 +25,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
# Ubuntu 24.04 ships GCC 13 by default, no upgrade needed
RUN pip3 install --break-system-packages conan==1.66.0
RUN pip3 install --break-system-packages conan==2.25.1
RUN echo "target arch $TARGETARCH"
RUN wget -qO- "https://cmake.org/files/v3.31/cmake-3.31.8-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
build/docker/builder/gpu/ubuntu20.04/Dockerfile
+1 -1
View File
@@ -25,7 +25,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
apt-get remove --purge -y && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install conan==1.66.0
RUN pip3 install conan==2.25.1
# Install openblas
# RUN wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
build/docker/builder/gpu/ubuntu22.04/Dockerfile
+1 -1
View File
@@ -18,7 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
# Install go
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.25.8.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go
# Install conan
RUN pip3 install conan==1.66.0
RUN pip3 install conan==2.25.1
# Install rust
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain=1.89 -y
ci/jenkins/PR-Arm.groovy
+1 -1
View File
@@ -61,7 +61,7 @@ pipeline {
set +a # stop automatically
docker run --net=host -v /root/.conan:/root/.conan -v \$(pwd):/root/milvus -w /root/milvus milvusdb/milvus-env:ubuntu20.04-\${DATE_VERSION} sh -c "make clean && make mode=RelWithDebInfo install"
docker run --net=host -v /root/.conan2:/root/.conan2 -v \$(pwd):/root/milvus -w /root/milvus milvusdb/milvus-env:ubuntu20.04-\${DATE_VERSION} sh -c "make clean && make mode=RelWithDebInfo install"
"""
withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
ci/jenkins/PublishArmBasedGPUImages.groovy
+1 -1
View File
@@ -45,7 +45,7 @@ pipeline {
. .env
set +a # stop automatically
docker run --net=host -v \$(pwd):/root/milvus -v /root/.conan:/root/.conan -w /root/milvus milvusdb/milvus-env:gpu-ubuntu22.04-\${GPU_DATE_VERSION} sh -c "make clean && make gpu-install"
docker run --net=host -v \$(pwd):/root/milvus -v /root/.conan2:/root/.conan2 -w /root/milvus milvusdb/milvus-env:gpu-ubuntu22.04-\${GPU_DATE_VERSION} sh -c "make clean && make gpu-install"
"""
withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
ci/jenkins/PublishArmBasedImages.groovy
+1 -1
View File
@@ -57,7 +57,7 @@ pipeline {
set +a # stop automatically
docker run --net=host -v /root/.conan:/root/.conan -v \$(pwd):/root/milvus -w /root/milvus milvusdb/milvus-env:ubuntu20.04-\${DATE_VERSION} sh -c "make clean && make install"
docker run --net=host -v /root/.conan2:/root/.conan2 -v \$(pwd):/root/milvus -w /root/milvus milvusdb/milvus-env:ubuntu20.04-\${DATE_VERSION} sh -c "make clean && make install"
"""
ci/jenkins/pod/rte-arm.yaml
+3 -3
View File
@@ -32,7 +32,7 @@ spec:
volumeMounts:
- mountPath: /var/run
name: docker-root
- mountPath: /root/.conan
- mountPath: /root/.conan2
name: build-cache
# - mountPath: /ci-logs
# name: ci-logs
@@ -51,13 +51,13 @@ spec:
volumeMounts:
- mountPath: /var/run
name: docker-root
- mountPath: /root/.conan
- mountPath: /root/.conan2
name: build-cache
volumes:
- emptyDir: {}
name: docker-root
- hostPath:
path: /root/.conan
path: /root/.conan2
type: DirectoryOrCreate
name: build-cache
# - name: ci-logs
docker-compose.yml
+4 -4
View File
@@ -29,7 +29,7 @@ services:
PULSAR_ADDRESS: ${PULSAR_ADDRESS}
ETCD_ENDPOINTS: ${ETCD_ENDPOINTS}
MINIO_ADDRESS: ${MINIO_ADDRESS}
CONAN_USER_HOME: /home/milvus
CONAN_HOME: /home/milvus/.conan2
AZURE_STORAGE_CONNECTION_STRING: ${AZURITE_CONNECTION_STRING}
ENABLE_GCP_NATIVE: ${ENABLE_GCP_NATIVE}
USE_ASAN: ${USE_ASAN}
@@ -38,7 +38,7 @@ services:
- ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-ccache:/ccache:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-go-mod:/go/pkg/mod:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-vscode-extensions:/home/milvus/.vscode-server/extensions:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan:/home/milvus/.conan:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan2:/home/milvus/.conan2:delegated
working_dir: '/go/src/github.com/milvus-io/milvus'
depends_on:
- etcd
@@ -72,7 +72,7 @@ services:
PULSAR_ADDRESS: ${PULSAR_ADDRESS}
ETCD_ENDPOINTS: ${ETCD_ENDPOINTS}
MINIO_ADDRESS: ${MINIO_ADDRESS}
CONAN_USER_HOME: /home/milvus
CONAN_HOME: /home/milvus/.conan2
AZURE_STORAGE_CONNECTION_STRING: ${AZURITE_CONNECTION_STRING}
ENABLE_GCP_NATIVE: ${ENABLE_GCP_NATIVE}
volumes: &builder-volumes-gpu
@@ -80,7 +80,7 @@ services:
- ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-ccache:/ccache:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-go-mod:/go/pkg/mod:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-vscode-extensions:/home/milvus/.vscode-server/extensions:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-conan:/home/milvus/.conan:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-conan2:/home/milvus/.conan2:delegated
working_dir: '/go/src/github.com/milvus-io/milvus'
depends_on:
- etcd
internal/core/CMakeLists.txt
+44 -4
View File
@@ -154,12 +154,52 @@ set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
set( CMAKE_BUILD_RPATH "${CMAKE_BINARY_DIR}/lib" )
# **************************** Dependencies ****************************
# Conan 2: CMakeToolchain sets CMAKE_PREFIX_PATH via toolchain file
# CMakeDeps generates <pkg>-config.cmake files for find_package()
list( APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/conan )
include( ${CMAKE_BINARY_DIR}/conan/conanbuildinfo.cmake )
set( CONAN_DISABLE_CHECK_COMPILER ON )
conan_basic_setup( KEEP_RPATHS )
set( CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CONAN_PROTOBUF_ROOT})
find_package(Protobuf REQUIRED)
find_package(Boost REQUIRED)
find_package(glog REQUIRED)
find_package(folly REQUIRED)
find_package(RocksDB REQUIRED)
find_package(Arrow REQUIRED)
find_package(gRPC REQUIRED)
find_package(opentelemetry-cpp REQUIRED)
find_package(google-cloud-cpp REQUIRED)
find_package(prometheus-cpp REQUIRED)
find_package(yaml-cpp REQUIRED)
find_package(nlohmann_json REQUIRED)
find_package(fmt REQUIRED)
find_package(re2 REQUIRED)
find_package(OpenSSL REQUIRED)
find_package(CURL REQUIRED)
find_package(ZLIB REQUIRED)
find_package(lz4 REQUIRED)
find_package(zstd REQUIRED)
find_package(Snappy REQUIRED)
find_package(roaring REQUIRED)
find_package(marisa REQUIRED)
find_package(geos REQUIRED)
find_package(ICU REQUIRED COMPONENTS uc)
find_package(xxHash REQUIRED)
find_package(TBB REQUIRED)
find_package(libsodium REQUIRED)
find_package(double-conversion REQUIRED)
find_package(gflags REQUIRED)
find_package(xsimd REQUIRED)
find_package(unordered_dense REQUIRED)
find_package(absl REQUIRED)
find_package(RapidJSON REQUIRED)
find_package(simde REQUIRED)
find_package(LibLZMA REQUIRED)
find_package(RdKafka REQUIRED)
find_package(GTest REQUIRED)
find_package(benchmark REQUIRED)
find_package(googleapis REQUIRED)
find_package(libavrocpp REQUIRED)
find_package(AWSSDK REQUIRED)
find_package(Azure REQUIRED)
include( CTest )
include( BuildUtils )
internal/core/conanfile.py
+121 -96
View File
@@ -1,108 +1,104 @@
from conans import ConanFile
required_conan_version = ">=2.0"
from conan import ConanFile
from conan.tools.cmake import CMakeDeps, CMakeToolchain
from conan.tools.files import copy
import os
class MilvusConan(ConanFile):
keep_imports = True
settings = "os", "compiler", "build_type", "arch"
requires = (
"rocksdb/6.29.5@milvus/dev#b1842a53ddff60240c5282a3da498ba1",
"boost/1.83.0@",
"onetbb/2021.9.0#4a223ff1b4025d02f31b65aedf5e7f4a",
"nlohmann_json/3.11.3#ffb9e9236619f1c883e36662f944345d",
"zstd/1.5.5#34e9debe03bf0964834a09dfbc31a5dd",
"lz4/1.9.4#c5afb86edd69ac0df30e3a9e192e43db",
"snappy/1.2.1#dfb7f7e837525210762d74e96eb4a3fc",
"arrow/17.0.0@milvus/dev-2.6#7af258a853e20887f9969f713110aac8",
"openssl/3.3.2#9f9f130d58e7c13e76bb8a559f0a6a8b",
"googleapis/cci.20221108#65604e1b3b9a6b363044da625b201a2a",
"gtest/1.13.0#f9548be18a41ccc6367efcb8146e92be",
"rocksdb/6.29.5@milvus/dev#67b8ae76ad7be5f779082f67416f89bf",
"onetbb/2021.9.0#f9d7a3aa294ac4a594a93f9b4c7f272d",
"zstd/1.5.5#70dc5eb8ea16708fc946fbac884c507e",
"lz4/1.9.4#7f0b5851453198536c14354ee30ca9ae",
"snappy/1.2.1#b940695c64ccbff63c1aabd4b1eee3f3",
"arrow/17.0.0@milvus/dev-2.6#c743ea7a6f2420ba5811b2be3df59892",
"libevent/2.1.12#95065aaefcd58d3956d6dfbfc5631d97",
"googleapis/cci.20221108#4553d68a2429cc0fff7d2bab4e5b3ea9",
"gtest/1.13.0#2cf98fac7337eb73fc4ee839dbcd4468",
"benchmark/1.7.0#459f3bb1a64400a886ba43047576df3c",
"protobuf/5.27.0@milvus/dev#6fff8583e2fe32babef04a9097f1d581",
"yaml-cpp/0.7.0#9c87b3998de893cf2e5a08ad09a7a6e0",
"marisa/0.2.6#68446854f5a420672d21f21191f8e5af",
"zlib/1.3.1#f52e03ae3d251dec704634230cd806a2",
"libcurl/8.10.1#a3113369c86086b0e84231844e7ed0a9",
"libevent/2.1.12#b6333a128075d75a3614bd8418bf2099",
"glog/0.7.1#83889ae482004d816ecac59dfef0d65a",
"fmt/11.0.2#5c7438ef4d5d69ab106a41e460ce11f3",
"gflags/2.2.2#b15c28c567c7ade7449cf994168a559f",
"double-conversion/3.3.0#33321c201741cc32b51169c6d2d05e60",
"yaml-cpp/0.7.0#355a88fb838abacfeb022dd52b91e248",
"marisa/0.2.6#a1352b20c0c6c48fee4968584ffef631",
"glog/0.7.1#a306e61d7b8311db8cb148ad62c48030",
"gflags/2.2.2#7671803f1dc19354cc90bd32874dcfda",
"double-conversion/3.3.0#640e35791a4bac95b0545e2f54b7aceb",
"libsodium/cci.20220430#7429a9e5351cc67bea3537229921714d",
"xsimd/9.0.1#ac9fd02a381698c4e08c5c4ca03b73e1",
"xz_utils/5.4.5#b885d1d79c9d30cff3803f7f551dbe66",
"xsimd/9.0.1#51df19a2d512f70597105ee2a2d21916",
"xz_utils/5.4.5#fc4e36861e0a47ecd4a40a00e6d29ac8",
"prometheus-cpp/1.2.4#0918d66c13f97acb7809759f9de49b3f",
"re2/20230301#f8efaf45f98d0193cd0b2ea08b6b4060",
"folly/2024.08.12.00@milvus/dev#e09fc71826ce6b4568441910665f0889",
"google-cloud-cpp/2.28.0@milvus/dev#25e69d743269d6c9ae5bf676af2174dc",
"opentelemetry-cpp/1.23.0@milvus/dev#bcd65b63b8db8447178ed93bbc94dcc0",
"librdkafka/1.9.1#e24dcbb0a1684dcf5a56d8d0692ceef3",
"abseil/20250127.0#e6c46096070cc3fd060e95a837cd3fb2",
"folly/2024.08.12.00@milvus/dev#f9b2bdf162c0ec47cb4e5404097b340d",
"google-cloud-cpp/2.28.0@milvus/dev#468918b43cec43624531a0340398cf43",
"opentelemetry-cpp/1.23.0@milvus/dev#11bc565ec6e82910ae8f7471da756720",
"librdkafka/1.9.1#ec1a00d5414f618555799be9566adfb7",
"roaring/3.0.0#25a703f80eda0764a31ef939229e202d",
"grpc/1.67.1@milvus/dev#00eeae5b14c7313dbc91f1a57c0a2554",
"rapidjson/cci.20230929#624c0094d741e6a3749d2e44d834b96c",
"crc32c/1.1.1",
"crc32c/1.1.2",
"simde/0.8.2#5e1edfd5cba92f25d79bf6ef4616b972",
"xxhash/0.8.3#199e63ab9800302c232d030b27accec0",
"xxhash/0.8.3#caa6d0af1b951c247922e38fbcebdbe6",
"unordered_dense/4.4.0#6a855c992618cc4c63019109a2e47298",
"geos/3.12.0#0b177c90c25a8ca210578fb9e2899c37",
"geos/3.12.0#a923af6dc4c18f87a7dfa960118f3166",
"icu/74.2#cd1937b9561b8950a2ae6311284c5813",
"libavrocpp/1.12.1.1@milvus/dev#a77043b1b435c3abef7b45710d05b300",
"libavrocpp/1.12.1.1@milvus/dev#cde7bb587a29f6f233bae7e18b71815d",
)
generators = ("cmake", "cmake_find_package")
default_options = {
"openssl:shared": True,
"openssl:no_apps": True,
"libevent:shared": True,
"double-conversion:shared": True,
"folly:shared": True,
"librdkafka:shared": True,
"librdkafka:zstd": True,
"librdkafka:ssl": True,
"librdkafka:sasl": True,
"rocksdb:shared": True,
"rocksdb:with_zstd": True,
"arrow:filesystem_layer": True,
"arrow:parquet": True,
"arrow:compute": True,
"arrow:with_re2": True,
"arrow:with_zstd": True,
"arrow:with_boost": True,
"arrow:with_thrift": True,
"arrow:with_jemalloc": False,
"arrow:with_openssl": True,
"arrow:shared": False,
"arrow:with_azure": True,
"arrow:with_s3": True,
"arrow:encryption": True,
"protobuf:shared": True,
"grpc:shared": True,
"grpc:secure": True,
"aws-sdk-cpp:config": True,
"aws-sdk-cpp:text-to-speech": False,
"aws-sdk-cpp:transfer": False,
"aws-sdk-cpp:s3-crt": True,
"gtest:build_gmock": True,
"boost:without_locale": False,
"boost:without_test": True,
"glog:with_gflags": True,
"glog:shared": True,
"prometheus-cpp:with_pull": False,
"fmt:header_only": False,
"onetbb:tbbmalloc": False,
"onetbb:tbbproxy": False,
"gflags:shared": True,
"gdal:shared": False,
"gdal:fPIC": True,
"icu:shared": False,
"icu:data_packaging": "library",
"xz_utils:shared": True,
"opentelemetry-cpp:with_stl": True,
"openssl/*:shared": True,
"openssl/*:no_apps": True,
"libevent/*:shared": True,
"double-conversion/*:shared": True,
"folly/*:shared": True,
"librdkafka/*:shared": True,
"librdkafka/*:zstd": True,
"librdkafka/*:ssl": True,
"librdkafka/*:sasl": True,
"rocksdb/*:shared": True,
"rocksdb/*:with_zstd": True,
"arrow/*:filesystem_layer": True,
"arrow/*:parquet": True,
"arrow/*:compute": True,
"arrow/*:with_re2": True,
"arrow/*:with_zstd": True,
"arrow/*:with_boost": True,
"arrow/*:with_thrift": True,
"arrow/*:with_jemalloc": False,
"arrow/*:with_openssl": True,
"arrow/*:shared": False,
"arrow/*:with_azure": True,
"arrow/*:with_s3": True,
"arrow/*:encryption": True,
"protobuf/*:shared": True,
"grpc/*:shared": True,
"grpc/*:secure": True,
"aws-sdk-cpp/*:config": True,
"aws-sdk-cpp/*:text-to-speech": False,
"aws-sdk-cpp/*:transfer": False,
"aws-sdk-cpp/*:s3-crt": True,
"gtest/*:build_gmock": True,
"boost/*:without_locale": False,
"boost/*:without_test": True,
"glog/*:with_gflags": True,
"glog/*:shared": True,
"prometheus-cpp/*:with_pull": False,
"fmt/*:header_only": False,
"onetbb/*:tbbmalloc": False,
"onetbb/*:tbbproxy": False,
"gflags/*:shared": True,
"gdal/*:shared": False,
"gdal/*:fPIC": True,
"icu/*:shared": False,
"icu/*:data_packaging": "library",
"xz_utils/*:shared": True,
"opentelemetry-cpp/*:with_stl": True,
}
def configure(self):
if self.settings.arch not in ("x86_64", "x86"):
del self.options["folly"].use_sse4_2
try:
del self.options["folly"].use_sse4_2
except Exception:
pass
if self.settings.os == "Macos":
# abseil static linking on macOS (previously shared for X86 compat)
self.options["arrow"].with_jemalloc = False
@@ -110,18 +106,47 @@ class MilvusConan(ConanFile):
self.options["libcurl"].with_ssl = "openssl"
def requirements(self):
# force=True: override transitive dependency versions (same behavior as Conan 1)
# In Conan 1.x, the consumer's requires implicitly override transitive versions.
# In Conan 2.x, force=True must be explicit.
self.requires("boost/1.83.0#4e8a94ac1b88312af95eded83cd81ca8", force=True)
self.requires("openssl/3.3.2#9f9f130d58e7c13e76bb8a559f0a6a8b", force=True)
self.requires("protobuf/5.27.0@milvus/dev#42f031a96d21c230a6e05bcac4bdd633", force=True)
self.requires("grpc/1.67.1@milvus/dev#efeaa484b59bffaa579004d5e82ec4fd", force=True)
self.requires("zlib/1.3.1#8045430172a5f8d56ba001b14561b4ea", force=True)
self.requires("libcurl/8.10.1#a3113369c86086b0e84231844e7ed0a9", force=True)
self.requires("nlohmann_json/3.11.3#ffb9e9236619f1c883e36662f944345d", force=True)
self.requires("abseil/20250127.0#481edcc75deb0efb16500f511f0f0a1c", force=True)
self.requires("fmt/11.0.2#eb98daa559c7c59d591f4720dde4cd5c", force=True)
self.requires("rapidjson/cci.20230929#0a3982e5f4fa453a9b9cd0dd5b1dcb3a", force=True)
self.requires("aws-sdk-cpp/1.11.692@milvus/dev#c309ce91fa572fff68f9f4e36d477a04")
if self.settings.os != "Macos":
self.requires("libunwind/1.8.1#97965ef7da98cf1662e14219b14134f7")
self.requires(
"aws-sdk-cpp/1.11.692@milvus/dev#1e17deac19383217d291a01c23147b33"
)
self.requires("libunwind/1.8.1#748a981ace010b80163a08867b732e71")
# Override s2n 1.4.1 (from aws-c-io) to 1.6.0 for OpenSSL 3.x FIPS detection
if self.settings.os in ["Linux", "FreeBSD"]:
self.requires("s2n/1.6.0")
self.requires("s2n/1.6.0#4fa3b751b92e126a55e45dce723f0384", force=True)
def imports(self):
self.copy("*.dylib", "../lib", "lib")
self.copy("*.dll", "../lib", "lib")
self.copy("*.so*", "../lib", "lib")
self.copy("*", "../bin", "bin")
self.copy("*.proto", "../include", "include")
def generate(self):
deps = CMakeDeps(self)
# Set cmake file names to match what downstream projects expect
deps.set_property("libavrocpp", "cmake_file_name", "libavrocpp")
deps.set_property("libavrocpp", "cmake_target_name", "libavrocpp::libavrocpp")
deps.generate()
tc = CMakeToolchain(self)
tc.generate()
# Copy shared libraries (replaces imports() from Conan 1)
# In Conan 1, imports() dst="lib" was relative to the build dir (cmake_build/).
# In Conan 2, we use generators_folder (cmake_build/conan/) parent to get cmake_build/lib.
build_dir = os.path.join(self.generators_folder, "..")
for dep in self.dependencies.values():
if dep.package_folder:
copy(self, "*.so*", src=os.path.join(dep.package_folder, "lib"),
dst=os.path.join(build_dir, "lib"))
copy(self, "*.dylib", src=os.path.join(dep.package_folder, "lib"),
dst=os.path.join(build_dir, "lib"))
copy(self, "*.dll", src=os.path.join(dep.package_folder, "lib"),
dst=os.path.join(build_dir, "lib"))
copy(self, "*", src=os.path.join(dep.package_folder, "bin"),
dst=os.path.join(build_dir, "bin"))
copy(self, "*.proto", src=os.path.join(dep.package_folder, "include"),
dst=os.path.join(build_dir, "include"))
internal/core/src/CMakeLists.txt
+60 -2
View File
@@ -33,11 +33,69 @@ include_directories(
${SIMDJSON_INCLUDE_DIR}
${TANTIVY_INCLUDE_DIR}
${BSONCXX_INCLUDE_DIR}
${CONAN_INCLUDE_DIRS}
${MILVUS_COMMON_INCLUDE_DIR}
${MILVUS_STORAGE_INCLUDE_DIR}
)
# Conan dependency targets shared between milvus_conan_deps (for object library
# include propagation) and LINK_TARGETS (for milvus_core final linking).
set(CONAN_TARGETS
protobuf::libprotobuf
glog::glog
gflags
Folly::folly
RocksDB::rocksdb-shared
arrow::arrow
gRPC::grpc++
opentelemetry-cpp::opentelemetry-cpp
google-cloud-cpp::google-cloud-cpp
prometheus-cpp::prometheus-cpp
yaml-cpp
nlohmann_json::nlohmann_json
fmt::fmt
re2::re2
OpenSSL::SSL
OpenSSL::Crypto
CURL::libcurl
ZLIB::ZLIB
lz4::lz4
zstd::libzstd_static
Snappy::snappy
roaring::roaring
marisa::marisa
GEOS::geos_c
ICU::uc
xxHash::xxhash
TBB::tbb
libsodium::libsodium
double-conversion::double-conversion
xsimd
unordered_dense::unordered_dense
abseil::abseil
rapidjson
simde::simde
LibLZMA::LibLZMA
RdKafka::rdkafka++
googleapis::googleapis
boost::boost
libavrocpp::libavrocpp
aws-sdk-cpp::aws-sdk-cpp
Azure::azure-core
)
# Interface library to propagate Conan include directories to object libraries.
# In Conan 1, conan_basic_setup() set global include dirs. In Conan 2, includes
# are per-target, so object libraries need explicit linkage to get include paths.
add_library(milvus_conan_deps INTERFACE)
target_link_libraries(milvus_conan_deps INTERFACE ${CONAN_TARGETS})
# Azure SDK recipe doesn't set includedirs on components in Conan 2.x.
# Locate azure headers via CMAKE_INCLUDE_PATH (set by conan_toolchain.cmake).
# NO_CACHE prevents stale NOTFOUND results across cmake reconfigures.
find_path(_AZURE_INCLUDE_DIR azure/core.hpp NO_CACHE)
if(_AZURE_INCLUDE_DIR)
target_include_directories(milvus_conan_deps INTERFACE "${_AZURE_INCLUDE_DIR}")
endif()
add_subdirectory( pb )
add_subdirectory( config )
add_subdirectory( common )
@@ -109,7 +167,7 @@ set(LINK_TARGETS
milvus-common
bsoncxx
${OpenMP_CXX_FLAGS}
${CONAN_LIBS}
${CONAN_TARGETS}
)
if(USE_OPENDAL)
internal/core/src/bitset/CMakeLists.txt
+2 -1
View File
@@ -13,7 +13,8 @@ set(BITSET_SRCS
detail/platform/dynamic.cpp
)
add_library(milvus_bitset)
add_library(milvus_bitset STATIC)
target_link_libraries(milvus_bitset PUBLIC milvus_conan_deps)
if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64")
list(APPEND BITSET_SRCS
internal/core/src/clustering/CMakeLists.txt
+1
View File
@@ -11,3 +11,4 @@
add_source_at_current_directory_recursively()
add_library(milvus_clustering OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_clustering PUBLIC milvus_conan_deps)
internal/core/src/common/CMakeLists.txt
+1
View File
@@ -11,3 +11,4 @@
add_source_at_current_directory_recursively()
add_library(milvus_common OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_common PUBLIC milvus_conan_deps)
internal/core/src/config/CMakeLists.txt
+2 -1
View File
@@ -15,4 +15,5 @@
# limitations under the License.
add_source_at_current_directory_recursively()
add_library(milvus_config OBJECT ${SOURCE_FILES})
add_library(milvus_config OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_config PUBLIC milvus_conan_deps)
internal/core/src/exec/CMakeLists.txt
+1
View File
@@ -11,6 +11,7 @@
add_source_at_current_directory_recursively()
add_library(milvus_exec OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_exec PUBLIC milvus_conan_deps)
# ── Per-arch SIMD compile flags for runtime dispatch ──
# Each SimdFilter*.cpp is compiled with arch-specific flags so xsimd
internal/core/src/futures/CMakeLists.txt
+2 -1
View File
@@ -10,4 +10,5 @@
# or implied. See the License for the specific language governing permissions and limitations under the License
add_source_at_current_directory_recursively()
add_library(milvus_futures OBJECT ${SOURCE_FILES})
add_library(milvus_futures OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_futures PUBLIC milvus_conan_deps)
internal/core/src/index/CMakeLists.txt
+1
View File
@@ -11,3 +11,4 @@
add_source_at_current_directory_recursively()
add_library(milvus_index OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_index PUBLIC milvus_conan_deps)
internal/core/src/indexbuilder/CMakeLists.txt
+2 -1
View File
@@ -10,4 +10,5 @@
# or implied. See the License for the specific language governing permissions and limitations under the License
add_source_at_current_directory_recursively()
add_library(milvus_indexbuilder OBJECT ${SOURCE_FILES})
add_library(milvus_indexbuilder OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_indexbuilder PUBLIC milvus_conan_deps)
internal/core/src/minhash/CMakeLists.txt
+1
View File
@@ -22,6 +22,7 @@ list(FILTER SOURCE_FILES EXCLUDE REGEX "fusion_compute/x86/.*")
list(FILTER SOURCE_FILES EXCLUDE REGEX "fusion_compute/arm/.*")
add_library(milvus_minhash OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_minhash PUBLIC milvus_conan_deps)
# Build strategy: Compile all SIMD variants unconditionally
# This ensures build/deploy machine separation - binary contains all variants
internal/core/src/monitor/CMakeLists.txt
+1
View File
@@ -11,3 +11,4 @@
add_source_at_current_directory_recursively()
add_library(milvus_monitor OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_monitor PUBLIC milvus_conan_deps)
internal/core/src/pb/CMakeLists.txt
+1
View File
@@ -13,3 +13,4 @@ find_package(Protobuf REQUIRED)
add_source_at_current_directory_recursively()
add_library(milvus_pb OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_pb PUBLIC milvus_conan_deps)
internal/core/src/plan/CMakeLists.txt
+2 -1
View File
@@ -10,4 +10,5 @@
# or implied. See the License for the specific language governing permissions and limitations under the License
add_source_at_current_directory_recursively()
add_library(milvus_plan OBJECT ${SOURCE_FILES})
add_library(milvus_plan OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_plan PUBLIC milvus_conan_deps)
internal/core/src/query/CMakeLists.txt
+1
View File
@@ -11,3 +11,4 @@
add_source_at_current_directory_recursively()
add_library(milvus_query OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_query PUBLIC milvus_conan_deps)
internal/core/src/rescores/CMakeLists.txt
+1
View File
@@ -11,3 +11,4 @@
add_source_at_current_directory_recursively()
add_library(milvus_rescores OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_rescores PUBLIC milvus_conan_deps)
internal/core/src/segcore/CMakeLists.txt
+1
View File
@@ -12,6 +12,7 @@
add_source_at_current_directory_recursively()
add_library(milvus_segcore OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_segcore PUBLIC milvus_conan_deps)
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CHECK_SORTED ON)
internal/core/src/storage/CMakeLists.txt
+1
View File
@@ -44,3 +44,4 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/loon_ffi)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/plugin)
add_library(milvus_storage OBJECT ${SOURCE_FILES})
target_link_libraries(milvus_storage PUBLIC milvus_conan_deps)
internal/core/thirdparty/boost_ext/CMakeLists.txt
Vendored
+1
View File
@@ -1,2 +1,3 @@
find_package(Boost REQUIRED)
add_library(boost_bitset_ext dynamic_bitset_ext.cpp)
target_link_libraries(boost_bitset_ext PUBLIC Boost::boost)
internal/core/thirdparty/knowhere/CMakeLists.txt
Vendored
+1 -1
View File
@@ -45,7 +45,7 @@ else ()
set(WITH_SVS OFF CACHE BOOL "" FORCE )
endif()
set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} )
# Conan 2.x: CMAKE_PREFIX_PATH is set by conan_toolchain.cmake
FetchContent_Declare(
knowhere
GIT_REPOSITORY ${GIT_REPOSITORY}
internal/core/thirdparty/milvus-common/CMakeLists.txt
Vendored
-2
View File
@@ -24,8 +24,6 @@ message(STATUS ${CMAKE_BUILD_TYPE})
set(WITH_JAEGER ON CACHE BOOL "" FORCE )
set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} )
FetchContent_Declare(
milvus-common
GIT_REPOSITORY ${GIT_REPOSITORY}
internal/core/thirdparty/milvus-storage/CMakeLists.txt
Vendored
+5 -14
View File
@@ -32,26 +32,17 @@ endif ()
# Disable benchmark for milvus-storage (we don't need it and it may have missing deps)
set(WITH_BENCHMARK OFF CACHE BOOL "" FORCE)
set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} ${CONAN_PROTOBUF_ROOT} )
# Make Conan's protoc visible to milvus-storage's find_package(Protobuf)
# Conan 2.x: CMAKE_PREFIX_PATH is set by conan_toolchain.cmake.
# find_package(Protobuf) sets Protobuf_PROTOC_EXECUTABLE automatically.
# Fallback: search CMAKE_BINARY_DIR/bin for protoc installed by conan imports.
find_program(_PROTOC_PROGRAM protoc
PATHS ${CONAN_BIN_DIRS_PROTOBUF} ${CONAN_BIN_DIRS}
"${CMAKE_BINARY_DIR}/bin"
PATHS "${CMAKE_BINARY_DIR}/bin"
NO_DEFAULT_PATH)
if(NOT _PROTOC_PROGRAM)
file(GLOB _PROTOC_CANDIDATES
"${CONAN_BIN_DIRS_PROTOBUF}/protoc-*"
"${CMAKE_BINARY_DIR}/bin/protoc-*")
if(_PROTOC_CANDIDATES)
list(SORT _PROTOC_CANDIDATES)
list(GET _PROTOC_CANDIDATES -1 _PROTOC_PROGRAM)
endif()
endif()
if(_PROTOC_PROGRAM)
message(STATUS "Setting Protobuf_PROTOC_EXECUTABLE for milvus-storage: ${_PROTOC_PROGRAM}")
set(Protobuf_PROTOC_EXECUTABLE "${_PROTOC_PROGRAM}" CACHE FILEPATH "" FORCE)
endif()
FetchContent_Declare(
milvus-storage
GIT_REPOSITORY ${GIT_REPOSITORY}
internal/core/thirdparty/rdkafka/CMakeLists.txt
Vendored
+3 -1
View File
@@ -1,3 +1,5 @@
milvus_add_pkg_config("rdkafka")
install(DIRECTORY "${CONAN_INCLUDE_DIRS_LIBRDKAFKA}/librdkafka/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/librdkafka/")
get_target_property(_RDKAFKA_INCLUDE_DIRS RdKafka::rdkafka INTERFACE_INCLUDE_DIRECTORIES)
list(GET _RDKAFKA_INCLUDE_DIRS 0 _RDKAFKA_INCLUDE_DIR)
install(DIRECTORY "${_RDKAFKA_INCLUDE_DIR}/librdkafka/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/librdkafka/")
internal/core/thirdparty/rocksdb/CMakeLists.txt
Vendored
+3 -1
View File
@@ -1,3 +1,5 @@
milvus_add_pkg_config("rocksdb")
install(DIRECTORY "${CONAN_INCLUDE_DIRS_ROCKSDB}/rocksdb/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/rocksdb/")
get_target_property(_ROCKSDB_INCLUDE_DIRS RocksDB::rocksdb-shared INTERFACE_INCLUDE_DIRECTORIES)
list(GET _ROCKSDB_INCLUDE_DIRS 0 _ROCKSDB_INCLUDE_DIR)
install(DIRECTORY "${_ROCKSDB_INCLUDE_DIR}/rocksdb/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/rocksdb/")
internal/core/unittest/CMakeLists.txt
+5 -6
View File
@@ -16,7 +16,6 @@ include_directories(
${KNOWHERE_INCLUDE_DIR}
${SIMDJSON_INCLUDE_DIR}
${TANTIVY_INCLUDE_DIR}
${CONAN_INCLUDE_DIRS}
${MILVUS_COMMON_INCLUDE_DIR}
${MILVUS_STORAGE_INCLUDE_DIR}
)
@@ -125,9 +124,10 @@ add_executable(all_tests
)
target_link_libraries(all_tests
gtest
gmock
GTest::gtest
GTest::gmock
milvus_core
milvus_conan_deps
knowhere
milvus-storage
milvus-common
@@ -167,8 +167,7 @@ endif()
target_link_libraries(bitset_test
milvus_bitset
gtest
${CONAN_LIBS}
GTest::gtest
)
if (LINUX)
# folly shared library requires libaio; use --no-as-needed to ensure
@@ -179,7 +178,7 @@ endif()
install(TARGETS bitset_test DESTINATION unittest)
add_executable(test_json_uint64 ${CMAKE_HOME_DIRECTORY}/src/exec/expression/ExprJsonUint64Test.cpp)
target_link_libraries(test_json_uint64 gtest gtest_main milvus_core knowhere milvus-storage milvus-common ${CONAN_LIBS})
target_link_libraries(test_json_uint64 GTest::gtest GTest::gtest_main milvus_core milvus_conan_deps knowhere milvus-storage milvus-common)
target_link_options(test_json_uint64 PRIVATE "-L${PLANPARSER_LIB_DIR}")
target_link_libraries(test_json_uint64 milvus-planparser-cpp)
set_target_properties(test_json_uint64 PROPERTIES BUILD_RPATH "${PLANPARSER_LIB_DIR}" INSTALL_RPATH "${PLANPARSER_LIB_DIR}")
internal/core/unittest/test_json_stats/CMakeLists.txt
+2 -2
View File
@@ -24,8 +24,8 @@ add_executable(json_stats_test
)
target_link_libraries(json_stats_test
gtest
gmock
GTest::gtest
GTest::gmock
milvus_core
milvus-storage
)
scripts/3rdparty_build.sh
+39 -31
View File
@@ -88,15 +88,20 @@ if [[ -f "$HOME/.local/bin/conan" ]]; then
export PATH="$HOME/.local/bin:$PATH"
fi
# Ensure conan version matches the required version (1.66.0)
# Ensure conan version matches the required version (2.25.1)
# CI Docker images may have an older version pre-installed
REQUIRED_CONAN_VERSION="1.66.0"
REQUIRED_CONAN_VERSION="2.25.1"
CURRENT_CONAN_VERSION=$(conan --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "0.0.0")
if [[ "${CURRENT_CONAN_VERSION}" != "${REQUIRED_CONAN_VERSION}" ]]; then
echo "Conan version mismatch: ${CURRENT_CONAN_VERSION} != ${REQUIRED_CONAN_VERSION}, upgrading..."
pip3 install conan==${REQUIRED_CONAN_VERSION} 2>/dev/null || pip install conan==${REQUIRED_CONAN_VERSION}
fi
# Ensure a default profile exists (Conan 2.x requires it; CI images may not have one)
if [[ ! -f "$(conan config home)/profiles/default" ]]; then
conan profile detect
fi
pushd ${BUILD_OUTPUT_DIR}
# Unset LD_PRELOAD and LD_LIBRARY_PATH to prevent jemalloc and other shared
@@ -116,7 +121,6 @@ unset LD_LIBRARY_PATH
unset DYLD_LIBRARY_PATH
unset DYLD_INSERT_LIBRARIES
export CONAN_REVISIONS_ENABLED=1
# Enable parallel downloads for faster dependency resolution
export CONAN_CPU_COUNT=${CONAN_CPU_COUNT:-$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 4)}
export CXXFLAGS="-Wno-error=address -Wno-error=deprecated-declarations -include cstdint"
@@ -132,11 +136,13 @@ fi
export CMAKE_POLICY_VERSION_MINIMUM=3.5
# Determine the Conan remote URL, using the environment variable if set, otherwise defaulting
CONAN_ARTIFACTORY_URL="${CONAN_ARTIFACTORY_URL:-https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local}"
# Always ensure the default-conan-local2 remote is the highest priority (index 0) to avoid conflicts with other remotes that may have the same packages
CONAN_ARTIFACTORY_URL="${CONAN_ARTIFACTORY_URL:-https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2}"
if [[ ! `conan remote list` == *default-conan-local* ]]; then
conan remote add default-conan-local $CONAN_ARTIFACTORY_URL
if conan remote list | grep -q default-conan-local2; then
conan remote remove default-conan-local2
fi
conan remote add default-conan-local2 $CONAN_ARTIFACTORY_URL --index 0
# Install a conan pre_build hook that sets LD_LIBRARY_PATH (Linux) or
# DYLD_LIBRARY_PATH (macOS) before each package build. This ensures that
@@ -144,23 +150,24 @@ fi
# when invoked during downstream package builds (opentelemetry-cpp, googleapis).
# Use "conan config home" to get the correct conan home directory, as ~ may
# differ from the conan home in CI containers.
CONAN_HOME_DIR=$(conan config home 2>/dev/null || echo "${CONAN_USER_HOME:-$HOME}/.conan")
mkdir -p "$CONAN_HOME_DIR/hooks"
cat > "$CONAN_HOME_DIR/hooks/fix_shared_lib_env.py" << 'HOOK_EOF'
CONAN_HOME_DIR=$(conan config home 2>/dev/null || echo "${CONAN_HOME:-$HOME/.conan2}")
mkdir -p "$CONAN_HOME_DIR/extensions/hooks"
cat > "$CONAN_HOME_DIR/extensions/hooks/hook_fix_shared_lib_env.py" << 'HOOK_EOF'
import os, platform
_saved_env = {}
def pre_build(output, conanfile, **kwargs):
def pre_build(conanfile, **kwargs):
"""Set library path before build so tools like grpc_cpp_plugin can find
shared libs (e.g. libprotoc.so). Saved state is restored in post_build
to avoid polluting the conan process environment."""
dep_lib_dirs = []
try:
for _, dep in conanfile.deps_cpp_info.dependencies:
for p in dep.lib_paths:
if os.path.isdir(p):
dep_lib_dirs.append(p)
for dep in conanfile.dependencies.values():
if dep.package_folder:
lib_dir = os.path.join(dep.package_folder, "lib")
if os.path.isdir(lib_dir):
dep_lib_dirs.append(lib_dir)
except Exception:
return
if not dep_lib_dirs:
@@ -170,9 +177,9 @@ def pre_build(output, conanfile, **kwargs):
existing = os.environ.get(env_var, "")
new_val = ":".join(dep_lib_dirs) + (":" + existing if existing else "")
os.environ[env_var] = new_val
output.info("Set %s for %d dependency lib dirs" % (env_var, len(dep_lib_dirs)))
conanfile.output.info("Set %s for %d dependency lib dirs" % (env_var, len(dep_lib_dirs)))
def post_build(output, conanfile, **kwargs):
def post_build(conanfile, **kwargs):
"""Restore library path after build to prevent conan process corruption."""
for env_var, old_val in _saved_env.items():
if old_val is None:
@@ -181,7 +188,6 @@ def post_build(output, conanfile, **kwargs):
os.environ[env_var] = old_val
_saved_env.clear()
HOOK_EOF
conan config set hooks.fix_shared_lib_env 2>/dev/null
unameOut="$(uname -s)"
case "${unameOut}" in
@@ -191,7 +197,7 @@ case "${unameOut}" in
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
echo "Using CXX: $CXX"
echo "Using CC: $CC"
CONAN_ARGS="--install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -u"
CONAN_ARGS="--output-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 -u"
# On macOS, Conan packages with shared libraries (protobuf, grpc) produce
# binaries (protoc, grpc_cpp_plugin) whose install rpaths don't include
@@ -201,12 +207,11 @@ case "${unameOut}" in
# Fix: install a Conan post_package hook that adds dependency rpaths to
# each package's binaries right after it is built (before the manifest is
# sealed), so every tool binary works the first time it is invoked.
mkdir -p "$CONAN_HOME_DIR/hooks"
conan config set hooks.fix_macos_rpaths 2>/dev/null
cat > "$CONAN_HOME_DIR/hooks/fix_macos_rpaths.py" << 'HOOK_EOF'
mkdir -p "$CONAN_HOME_DIR/extensions/hooks"
cat > "$CONAN_HOME_DIR/extensions/hooks/hook_fix_macos_rpaths.py" << 'HOOK_EOF'
import os, platform, subprocess, stat
def post_package(output, conanfile, conanfile_path, **kwargs):
def post_package(conanfile, **kwargs):
if platform.system() != "Darwin":
return
pkg = conanfile.package_folder
@@ -217,10 +222,11 @@ def post_package(output, conanfile, conanfile_path, **kwargs):
return
dep_lib_dirs = []
try:
for _, dep in conanfile.deps_cpp_info.dependencies:
for p in dep.lib_paths:
if os.path.isdir(p):
dep_lib_dirs.append(p)
for dep in conanfile.dependencies.values():
if dep.package_folder:
lib_dir = os.path.join(dep.package_folder, "lib")
if os.path.isdir(lib_dir):
dep_lib_dirs.append(lib_dir)
except Exception:
return
if not dep_lib_dirs:
@@ -234,11 +240,13 @@ def post_package(output, conanfile, conanfile_path, **kwargs):
if "Mach-O" not in r.stdout:
continue
for lib_path in dep_lib_dirs:
subprocess.run(["install_name_tool", "-add_rpath", lib_path, fp],
capture_output=True)
r = subprocess.run(["install_name_tool", "-add_rpath", lib_path, fp],
capture_output=True, text=True)
if r.returncode != 0:
conanfile.output.warning("install_name_tool failed for %s: %s" % (name, r.stderr.strip()))
fixed.append(name)
if fixed:
output.info("Fixed macOS rpaths for: %s" % ", ".join(fixed))
conanfile.output.info("Fixed macOS rpaths for: %s" % ", ".join(fixed))
HOOK_EOF
conan install ${CPP_SRC_DIR} ${CONAN_ARGS} || { echo 'conan install failed'; exit 1; }
@@ -253,9 +261,9 @@ HOOK_EOF
export CPU_TARGET=avx
GCC_VERSION=`gcc -dumpversion`
if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; }
conan install ${CPP_SRC_DIR} --output-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; }
else
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -s compiler.cppstd=17 -u || { echo 'conan install failed'; exit 1; }
conan install ${CPP_SRC_DIR} --output-folder conan --build=missing -s build_type=${BUILD_TYPE} -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; }
fi
;;
*)
scripts/core_build.sh
+1
View File
@@ -240,6 +240,7 @@ export CMAKE_POLICY_VERSION_MINIMUM=3.5
arch=$(uname -m)
CMAKE_CMD="cmake \
${CMAKE_EXTRA_ARGS} \
-DCMAKE_TOOLCHAIN_FILE=${BUILD_OUTPUT_DIR}/conan/conan_toolchain.cmake \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
scripts/install_deps.sh
+2 -5
View File
@@ -57,7 +57,7 @@ print_error() {
# Minimum version requirements
MIN_CMAKE_VERSION="3.26"
MIN_GO_VERSION="1.21"
CONAN_VERSION="1.66.0"
CONAN_VERSION="2.25.1"
RUST_VERSION="1.89"
#######################################
@@ -355,10 +355,7 @@ install_rocky_deps() {
gcc_ver=$(gcc -dumpversion 2>/dev/null | cut -d. -f1)
if [ -n "$gcc_ver" ]; then
print_info "Configuring Conan profile for GCC ${gcc_ver}..."
conan profile new default --detect --force 2>/dev/null || true
conan profile update settings.compiler.version="$gcc_ver" default
conan profile update settings.compiler.libcxx=libstdc++11 default
conan profile update settings.compiler.cppstd=17 default
conan profile detect --force 2>/dev/null || true
fi
# Install Rust
scripts/install_deps_msys.sh
+1 -1
View File
@@ -23,7 +23,7 @@ pacmanInstall()
mingw-w64-x86_64-diffutils \
mingw-w64-x86_64-go
pip3 install conan==1.66.0
pip3 install conan==2.25.1
}
updateKey()
scripts/run_cpp_codecov.sh
+1 -1
View File
@@ -103,7 +103,7 @@ ${LCOV_CMD} -r "${FILE_INFO_COMBINE}" -o "${FILE_INFO_OUTPUT}" \
"*/unittest/*" \
"*/thirdparty/*" \
"*/3rdparty_download/*" \
"*/.conan/data/*"
"*/.conan2/*"
# generate html report
${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/
scripts/setenv.sh
+3 -7
View File
@@ -60,18 +60,14 @@ case "${unameOut}" in
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$MILVUS_LIB_DIRS"
export RPATH=$MILVUS_LIB_DIRS;;
Darwin*)
# detect llvm version by valid list (supports LLVM 15-17)
# Note: LLVM 18 is NOT supported because Conan 1.x cannot handle the newer
# compiler profiles/settings that LLVM 18 requires. Until we migrate to Conan 2,
# please use LLVM 15-17.
for llvm_version in 17 16 15 NOT_FOUND ; do
# detect llvm version by valid list (supports LLVM 14-18)
for llvm_version in 18 17 16 15 14 NOT_FOUND ; do
if brew ls --versions llvm@${llvm_version} > /dev/null 2>&1; then
break
fi
done
if [ "${llvm_version}" = "NOT_FOUND" ] ; then
echo "ERROR: Valid LLVM (15-17) not installed. Run: brew install llvm@17"
echo "NOTE: LLVM 18 is not supported due to Conan 1.x incompatibility."
echo "ERROR: Valid LLVM (14-18) not installed. Run: brew install llvm@17"
exit 1
fi
llvm_prefix="$(brew --prefix llvm@${llvm_version})"