mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-20 09:45:43 +00:00
enhance: remove unused C++ dependencies (rapidjson, OpenDAL backend) (#50552)
## What this PR does Removes two unused C++ third-party dependencies from `internal/core` to reduce dependency count, comprehension cost, and link/build surface. ### rapidjson Linked into `milvus_core` but has **zero `#include` usage** in the C++ sources (only a stale comment in `common/Json.h`). knowhere v3.0.3 and milvus-storage do not depend on it. Removes the conan requirement, `find_package(RapidJSON)`, and the link target. ### OpenDAL backend Experimental `ChunkManager` backend gated behind `USE_OPENDAL` (**default OFF**) — released binaries never built it. Removes `OpenDALChunkManager`, the vendored OpenDAL Rust dependency, and all `USE_OPENDAL` build plumbing (Makefile / core_build.sh / 3rdparty_build.sh / CMake). The Go-side `"opendal"` storageType value remains a backward-compatible alias to the standard remote chunk manager. ### Kept `simde` is **not** removed — knowhere v3.0.3 requires it (`find_package(simde REQUIRED)`, used in `src/index/sparse/codec/varintdecode.c`). issue: #50551 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Signed-off-by: xiaofanluan <xf@hjjaq.com> Co-authored-by: xiaofanluan <xf@hjjaq.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
James
co-authored by
xiaofanluan
Claude Opus 4.8
xiaofanluan
Claude Opus 4.8
parent
0cf0fd256a
commit
6492fa011b
@@ -53,10 +53,6 @@ ifdef TANTIVY_FEATURES
|
||||
tantivy_features = ${TANTIVY_FEATURES}
|
||||
endif
|
||||
|
||||
use_opendal = OFF
|
||||
ifdef USE_OPENDAL
|
||||
use_opendal = ${USE_OPENDAL}
|
||||
endif
|
||||
|
||||
use_svs = OFF
|
||||
ifdef USE_SVS
|
||||
@@ -286,7 +282,7 @@ download-milvus-proto:
|
||||
|
||||
build-3rdparty:
|
||||
@echo "Build 3rdparty ..."
|
||||
@(env bash $(PWD)/scripts/3rdparty_build.sh -o ${use_opendal} -t ${mode})
|
||||
@(env bash $(PWD)/scripts/3rdparty_build.sh -t ${mode})
|
||||
|
||||
generated-proto-without-cpp: download-milvus-proto get-proto-deps
|
||||
@echo "Generate proto ..."
|
||||
@@ -298,19 +294,19 @@ generated-proto: download-milvus-proto build-3rdparty get-proto-deps
|
||||
|
||||
build-cpp: generated-proto plan-parser-lib
|
||||
@echo "Building Milvus cpp library ..."
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features) -S ${use_svs})
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -f $(tantivy_features) -S ${use_svs})
|
||||
|
||||
build-cpp-gpu: generated-proto plan-parser-lib
|
||||
@echo "Building Milvus cpp gpu library ... "
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features) -S ${use_svs})
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -f $(tantivy_features) -S ${use_svs})
|
||||
|
||||
build-cpp-with-unittest: generated-proto plan-parser-lib
|
||||
@echo "Building Milvus cpp library with unittest ... "
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features) -S ${use_svs})
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -f $(tantivy_features) -S ${use_svs})
|
||||
|
||||
build-cpp-with-coverage: generated-proto plan-parser-lib
|
||||
@echo "Building Milvus cpp library with coverage and unittest ..."
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -c -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -o ${use_opendal} -f $(tantivy_features) -S ${use_svs})
|
||||
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -c -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine} -f $(tantivy_features) -S ${use_svs})
|
||||
|
||||
check-proto-product: generated-proto
|
||||
@(env bash $(PWD)/scripts/check_proto_product.sh)
|
||||
|
||||
+1
-1
@@ -991,7 +991,7 @@ common:
|
||||
gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
|
||||
parquetStatsSkipIndex:
|
||||
enabled: false # whether to skip parquet stats index when reading; set true to enable skipping.
|
||||
storageType: remote # please adjust in embedded Milvus: local, available values are [local, remote, opendal], value minio is deprecated, use remote instead
|
||||
storageType: remote # please adjust in embedded Milvus: local, available values are [local, remote], value minio is deprecated, use remote instead
|
||||
storage:
|
||||
manifestTransactionRetryLimit: 10 # Maximum number of retry attempts for V3 storage manifest transaction commits on optimistic concurrency conflicts
|
||||
stv2:
|
||||
|
||||
@@ -37,10 +37,6 @@ if ( USE_DYNAMIC_SIMD )
|
||||
add_definitions(-DUSE_DYNAMIC_SIMD)
|
||||
endif()
|
||||
|
||||
if (USE_OPENDAL)
|
||||
add_definitions(-DUSE_OPENDAL)
|
||||
endif()
|
||||
|
||||
project(core)
|
||||
include(CheckCXXCompilerFlag)
|
||||
if ( APPLE )
|
||||
@@ -190,7 +186,6 @@ find_package(gflags REQUIRED)
|
||||
find_package(xsimd REQUIRED)
|
||||
find_package(unordered_dense REQUIRED)
|
||||
find_package(absl REQUIRED)
|
||||
find_package(RapidJSON REQUIRED)
|
||||
find_package(simde REQUIRED)
|
||||
find_package(milvus-common REQUIRED)
|
||||
if (LINUX)
|
||||
|
||||
@@ -123,7 +123,6 @@ class MilvusConan(ConanFile):
|
||||
self.requires("nlohmann_json/3.11.3#ffb9e9236619f1c883e36662f944345d", force=True)
|
||||
self.requires("abseil/20250127.0#481edcc75deb0efb16500f511f0f0a1c", force=True)
|
||||
self.requires("fmt/11.2.0#eb98daa559c7c59d591f4720dde4cd5c", force=True)
|
||||
self.requires("rapidjson/cci.20230929#0a3982e5f4fa453a9b9cd0dd5b1dcb3a", force=True)
|
||||
# azure-sdk-for-cpp is a transitive dep of Arrow, but must be declared
|
||||
# as a direct dep so CMakeDeps generates standalone cmake config files.
|
||||
# Without this, find_package(Azure) can't find include directories.
|
||||
|
||||
@@ -71,7 +71,6 @@ set(CONAN_TARGETS
|
||||
xsimd
|
||||
unordered_dense::unordered_dense
|
||||
abseil::abseil
|
||||
rapidjson
|
||||
simde::simde
|
||||
milvus-common::milvus-common
|
||||
LibLZMA::LibLZMA
|
||||
@@ -169,10 +168,6 @@ set(LINK_TARGETS
|
||||
${CONAN_TARGETS}
|
||||
)
|
||||
|
||||
if(USE_OPENDAL)
|
||||
set(LINK_TARGETS ${LINK_TARGETS} opendal)
|
||||
endif()
|
||||
|
||||
if(ENABLE_AZURE_FS)
|
||||
set(AZURE_BUILD_DIR ON)
|
||||
# Arrow already includes Azure SDK when built with_azure=True
|
||||
|
||||
@@ -32,11 +32,6 @@ if (ENABLE_AZURE_FS)
|
||||
add_subdirectory(azure)
|
||||
endif()
|
||||
|
||||
if(USE_OPENDAL)
|
||||
include_directories(${OPENDAL_INCLUDE_DIR})
|
||||
set(SOURCE_FILES ${SOURCE_FILES} opendal/OpenDALChunkManager.cpp)
|
||||
endif()
|
||||
|
||||
# Add loon_ffi subdirectory to include loon FFI source files
|
||||
add_subdirectory(loon_ffi)
|
||||
|
||||
|
||||
@@ -137,7 +137,6 @@ enum class ChunkManagerType : int8_t {
|
||||
Local = 1,
|
||||
Minio = 2,
|
||||
Remote = 3,
|
||||
OpenDAL = 4,
|
||||
};
|
||||
|
||||
extern std::map<std::string, ChunkManagerType> ChunkManagerType_Map;
|
||||
|
||||
@@ -50,9 +50,6 @@
|
||||
#include "storage/LocalChunkManager.h"
|
||||
#include "storage/MemFileManagerImpl.h"
|
||||
#include "storage/minio/MinioChunkManager.h"
|
||||
#ifdef USE_OPENDAL
|
||||
#include "storage/opendal/OpenDALChunkManager.h"
|
||||
#endif
|
||||
#include "storage/Types.h"
|
||||
#include "storage/Util.h"
|
||||
#include "common/Common.h"
|
||||
@@ -82,8 +79,7 @@ constexpr const char* TEMP = "tmp";
|
||||
std::map<std::string, ChunkManagerType> ChunkManagerType_Map = {
|
||||
{"local", ChunkManagerType::Local},
|
||||
{"minio", ChunkManagerType::Minio},
|
||||
{"remote", ChunkManagerType::Remote},
|
||||
{"opendal", ChunkManagerType::OpenDAL}};
|
||||
{"remote", ChunkManagerType::Remote}};
|
||||
|
||||
static std::shared_ptr<arrow::Array>
|
||||
NormalizeExternalArrowByType(const std::shared_ptr<arrow::Array>& array,
|
||||
@@ -1180,11 +1176,6 @@ CreateChunkManager(const StorageConfig& storage_config) {
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifdef USE_OPENDAL
|
||||
case ChunkManagerType::OpenDAL: {
|
||||
return std::make_shared<OpenDALChunkManager>(storage_config);
|
||||
}
|
||||
#endif
|
||||
default: {
|
||||
ThrowInfo(ConfigInvalid,
|
||||
"unsupported storage_config.storage_type {}",
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
|
||||
#include "log/Log.h"
|
||||
#include "opendal.h"
|
||||
#include "common/EasyAssert.h"
|
||||
#include "storage/Util.h"
|
||||
#include "storage/opendal/OpenDALChunkManager.h"
|
||||
|
||||
namespace milvus::storage {
|
||||
|
||||
std::string
|
||||
ToString(opendal_bytes* bs) {
|
||||
return {reinterpret_cast<const char*>(bs->data), bs->len};
|
||||
}
|
||||
|
||||
#define THROWOPENDALERROR(err, msg) \
|
||||
do { \
|
||||
auto exception = SegcoreError( \
|
||||
S3Error, fmt::format("{}: {}", (msg), ToString(&(err)->message))); \
|
||||
opendal_error_free(err); \
|
||||
throw exception; \
|
||||
} while (0)
|
||||
|
||||
// std::once_flag init_flag_;
|
||||
|
||||
OpenDALChunkManager::OpenDALChunkManager(const StorageConfig& storage_config)
|
||||
: default_bucket_name_(storage_config.bucket_name) {
|
||||
// std::call_once(init_flag_, []() { opendal_init_logger(); });
|
||||
remote_root_path_ = storage_config.root_path;
|
||||
std::string storageType;
|
||||
if (storage_config.cloud_provider == "gcp") {
|
||||
storageType = "gcs";
|
||||
} else if (storage_config.cloud_provider == "aliyun") {
|
||||
storageType = "oss";
|
||||
} else if (storage_config.cloud_provider == "azure") {
|
||||
storageType = "azblob";
|
||||
} else {
|
||||
storageType = "s3";
|
||||
}
|
||||
|
||||
opendal_operator_options* op_options_ = opendal_operator_options_new();
|
||||
if (!storage_config.access_key_id.empty() &&
|
||||
!storage_config.access_key_value.empty()) {
|
||||
opendal_operator_options_set(
|
||||
op_options_, "access_key_id", storage_config.access_key_id.c_str());
|
||||
opendal_operator_options_set(op_options_,
|
||||
"secret_access_key",
|
||||
storage_config.access_key_value.c_str());
|
||||
storageType = "s3";
|
||||
}
|
||||
opendal_operator_options_set(op_options_, "root", "/");
|
||||
opendal_operator_options_set(
|
||||
op_options_, "bucket", storage_config.bucket_name.c_str());
|
||||
opendal_operator_options_set(op_options_,
|
||||
"region",
|
||||
storage_config.region.empty()
|
||||
? "us-east-1"
|
||||
: storage_config.region.c_str());
|
||||
opendal_operator_options_set(
|
||||
op_options_,
|
||||
"endpoint",
|
||||
((storage_config.useSSL ? "https://" : "http://") +
|
||||
storage_config.address)
|
||||
.c_str());
|
||||
opendal_operator_options_set(
|
||||
op_options_,
|
||||
"enable_virtual_host_style",
|
||||
storage_config.useVirtualHost ? "true" : "false");
|
||||
|
||||
auto op = opendal_operator_new(storageType.c_str(), op_options_);
|
||||
if (op.error != nullptr) {
|
||||
THROWOPENDALERROR(op.error, "Init opendal error");
|
||||
}
|
||||
op_ptr_ = op.op;
|
||||
opendal_operator_options_free(op_options_);
|
||||
LOG_INFO(
|
||||
"init OpenDALChunkManager with "
|
||||
"parameter[endpoint={}][bucket_name={}][root_path={}][use_secure={}]",
|
||||
storage_config.address,
|
||||
storage_config.bucket_name,
|
||||
storage_config.root_path,
|
||||
storage_config.useSSL);
|
||||
}
|
||||
|
||||
OpenDALChunkManager::~OpenDALChunkManager() {
|
||||
opendal_operator_free(op_ptr_);
|
||||
}
|
||||
|
||||
uint64_t
|
||||
OpenDALChunkManager::Size(const std::string& filepath) {
|
||||
auto ret = opendal_operator_stat(op_ptr_, filepath.c_str());
|
||||
if (ret.error != nullptr) {
|
||||
THROWOPENDALERROR(ret.error, "GetObjectSize");
|
||||
}
|
||||
auto size = opendal_metadata_content_length(ret.meta);
|
||||
opendal_metadata_free(ret.meta);
|
||||
return size;
|
||||
}
|
||||
|
||||
bool
|
||||
OpenDALChunkManager::Exist(const std::string& filepath) {
|
||||
auto ret = opendal_operator_is_exist(op_ptr_, filepath.c_str());
|
||||
if (ret.error != nullptr) {
|
||||
THROWOPENDALERROR(ret.error, "ObjectExists");
|
||||
}
|
||||
return ret.is_exist;
|
||||
}
|
||||
|
||||
void
|
||||
OpenDALChunkManager::Remove(const std::string& filepath) {
|
||||
auto ret = opendal_operator_delete(op_ptr_, filepath.c_str());
|
||||
if (ret != nullptr) {
|
||||
THROWOPENDALERROR(ret, "RemoveObject");
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
OpenDALChunkManager::ListWithPrefix(const std::string& filepath) {
|
||||
auto ret = opendal_operator_list(op_ptr_, filepath.c_str());
|
||||
if (ret.error != nullptr) {
|
||||
THROWOPENDALERROR(ret.error, "ListObjects");
|
||||
}
|
||||
auto lister = OpendalLister(ret.lister);
|
||||
std::vector<std::string> objects;
|
||||
opendal_result_lister_next result = opendal_lister_next(lister.Get());
|
||||
if (result.error != nullptr) {
|
||||
THROWOPENDALERROR(result.error, "ListObjects");
|
||||
}
|
||||
auto entry = result.entry;
|
||||
while (entry) {
|
||||
const char* de_path = opendal_entry_path(entry);
|
||||
objects.push_back(std::string(de_path));
|
||||
opendal_entry_free(entry);
|
||||
result = opendal_lister_next(lister.Get());
|
||||
if (result.error != nullptr) {
|
||||
THROWOPENDALERROR(result.error, "ListObjects");
|
||||
}
|
||||
entry = result.entry;
|
||||
}
|
||||
return objects;
|
||||
}
|
||||
|
||||
uint64_t
|
||||
OpenDALChunkManager::Read(const std::string& filepath,
|
||||
void* buf,
|
||||
uint64_t size) {
|
||||
auto ret = opendal_operator_reader(op_ptr_, filepath.c_str());
|
||||
if (ret.error != nullptr) {
|
||||
THROWOPENDALERROR(ret.error, "GetObjectBuffer");
|
||||
}
|
||||
auto reader = OpendalReader(ret.reader);
|
||||
uint64_t buf_size = 16 * 1024;
|
||||
uint64_t buf_index = 0;
|
||||
while (true) {
|
||||
auto read_ret =
|
||||
opendal_reader_read(reader.Get(),
|
||||
reinterpret_cast<uint8_t*>(buf) + buf_index,
|
||||
buf_size);
|
||||
buf_index += read_ret.size;
|
||||
if (read_ret.error != nullptr) {
|
||||
THROWOPENDALERROR(read_ret.error, "GetObjectBuffer");
|
||||
}
|
||||
if (read_ret.size == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (buf_index != size) {
|
||||
ThrowInfo(
|
||||
S3Error,
|
||||
fmt::format(
|
||||
"Read size mismatch, target size is {}, actual size is {}",
|
||||
size,
|
||||
buf_index));
|
||||
}
|
||||
return buf_index;
|
||||
}
|
||||
|
||||
void
|
||||
OpenDALChunkManager::Write(const std::string& filepath,
|
||||
void* buf,
|
||||
uint64_t size) {
|
||||
auto ret = opendal_operator_write(
|
||||
op_ptr_, filepath.c_str(), {reinterpret_cast<uint8_t*>(buf), size});
|
||||
if (ret != nullptr) {
|
||||
THROWOPENDALERROR(ret, "Write");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace milvus::storage
|
||||
@@ -1,121 +0,0 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "common/EasyAssert.h"
|
||||
#include "storage/ChunkManager.h"
|
||||
#include "storage/Types.h"
|
||||
#include "opendal.h"
|
||||
|
||||
namespace milvus::storage {
|
||||
class OpenDALChunkManager : public ChunkManager {
|
||||
public:
|
||||
OpenDALChunkManager() = default;
|
||||
explicit OpenDALChunkManager(const StorageConfig& storage_config);
|
||||
|
||||
OpenDALChunkManager(const OpenDALChunkManager&);
|
||||
OpenDALChunkManager&
|
||||
operator=(const OpenDALChunkManager&);
|
||||
|
||||
public:
|
||||
virtual ~OpenDALChunkManager();
|
||||
|
||||
bool
|
||||
Exist(const std::string& filepath) override;
|
||||
|
||||
uint64_t
|
||||
Size(const std::string& filepath) override;
|
||||
|
||||
uint64_t
|
||||
Read(const std::string& filepath,
|
||||
uint64_t offset,
|
||||
void* buf,
|
||||
uint64_t len) override {
|
||||
ThrowInfo(NotImplemented, GetName() + "Read with offset not implement");
|
||||
}
|
||||
|
||||
void
|
||||
Write(const std::string& filepath,
|
||||
uint64_t offset,
|
||||
void* buf,
|
||||
uint64_t len) override {
|
||||
ThrowInfo(NotImplemented,
|
||||
GetName() + "Write with offset not implement");
|
||||
}
|
||||
|
||||
uint64_t
|
||||
Read(const std::string& filepath, void* buf, uint64_t len) override;
|
||||
|
||||
void
|
||||
Write(const std::string& filepath, void* buf, uint64_t len) override;
|
||||
|
||||
std::vector<std::string>
|
||||
ListWithPrefix(const std::string& filepath) override;
|
||||
|
||||
void
|
||||
Remove(const std::string& filepath) override;
|
||||
|
||||
std::string
|
||||
GetName() const override {
|
||||
return "OpenDALChunkManager";
|
||||
}
|
||||
|
||||
std::string
|
||||
GetRootPath() const override {
|
||||
return remote_root_path_;
|
||||
}
|
||||
|
||||
std::string
|
||||
GetBucketName() const override {
|
||||
return default_bucket_name_;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string default_bucket_name_;
|
||||
std::string remote_root_path_;
|
||||
|
||||
const opendal_operator* op_ptr_;
|
||||
};
|
||||
struct OpendalReader {
|
||||
explicit OpendalReader(opendal_reader* reader) : reader_(reader) {
|
||||
}
|
||||
~OpendalReader() {
|
||||
opendal_reader_free(reader_);
|
||||
}
|
||||
opendal_reader*
|
||||
Get() {
|
||||
return reader_;
|
||||
}
|
||||
|
||||
private:
|
||||
opendal_reader* reader_;
|
||||
};
|
||||
struct OpendalLister {
|
||||
explicit OpendalLister(opendal_lister* lister) : lister_(lister) {
|
||||
}
|
||||
~OpendalLister() {
|
||||
opendal_lister_free(lister_);
|
||||
}
|
||||
opendal_lister*
|
||||
Get() {
|
||||
return lister_;
|
||||
}
|
||||
|
||||
private:
|
||||
opendal_lister* lister_;
|
||||
};
|
||||
} // namespace milvus::storage
|
||||
-3
@@ -36,9 +36,6 @@ add_subdirectory(rocksdb)
|
||||
add_subdirectory(rdkafka)
|
||||
add_subdirectory(simdjson)
|
||||
add_subdirectory(bsoncxx)
|
||||
if (USE_OPENDAL)
|
||||
add_subdirectory(opendal)
|
||||
endif()
|
||||
add_subdirectory(tantivy)
|
||||
|
||||
if (LINUX)
|
||||
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
#-------------------------------------------------------------------------------
|
||||
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
# or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
message(STATUS "Building (vendored) opendal from source")
|
||||
|
||||
set(GIT_REPOSITORY "https://github.com/apache/opendal.git")
|
||||
set(GIT_TAG "v0.43.0-rc.2")
|
||||
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
set(CARGO_CMD cargo +1.89 build --verbose)
|
||||
set(TARGET_DIR "debug")
|
||||
else ()
|
||||
set(CARGO_CMD cargo +1.89 build --release --verbose)
|
||||
set(TARGET_DIR "release")
|
||||
endif ()
|
||||
|
||||
set(SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/src")
|
||||
|
||||
FetchContent_Declare(
|
||||
opendal
|
||||
GIT_REPOSITORY ${GIT_REPOSITORY}
|
||||
GIT_TAG ${GIT_TAG}
|
||||
GIT_SHALLOW TRUE
|
||||
SOURCE_DIR ${SOURCE_DIR}
|
||||
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH})
|
||||
|
||||
FetchContent_GetProperties(opendal)
|
||||
if ( NOT opendal_POPULATED )
|
||||
FetchContent_Populate(opendal)
|
||||
endif()
|
||||
|
||||
set(OPENDAL_LIB_DIR "${SOURCE_DIR}/target/${TARGET_DIR}" CACHE INTERNAL "opendal lib dir")
|
||||
set(OPENDAL_INCLUDE_DIR "${SOURCE_DIR}/bindings/c/include" CACHE INTERNAL "opendal include dir")
|
||||
set(OPENDAL_LIB "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE INTERNAL "opendal lib")
|
||||
|
||||
add_custom_target(build_opendal
|
||||
COMMAND ${CARGO_CMD}
|
||||
WORKING_DIRECTORY ${SOURCE_DIR}/bindings/c
|
||||
)
|
||||
|
||||
add_library(opendal STATIC IMPORTED)
|
||||
add_dependencies(opendal build_opendal)
|
||||
set_target_properties(opendal
|
||||
PROPERTIES
|
||||
IMPORTED_GLOBAL TRUE
|
||||
IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${OPENDAL_INCLUDE_DIR}")
|
||||
|
||||
get_target_property(OPENDAL_IMPORTED_LOCATION opendal IMPORTED_LOCATION)
|
||||
get_target_property(OPENDAL_INTERFACE_INCLUDE_DIRECTORIES opendal INTERFACE_INCLUDE_DIRECTORIES)
|
||||
message("OPENDAL_IMPORTED_LOCATION: ${OPENDAL_IMPORTED_LOCATION}")
|
||||
message("OPENDAL_INTERFACE_INCLUDE_DIRECTORIES: ${OPENDAL_INTERFACE_INCLUDE_DIRECTORIES}")
|
||||
@@ -662,7 +662,7 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
|
||||
Key: "common.storageType",
|
||||
Version: "2.0.0",
|
||||
DefaultValue: "remote",
|
||||
Doc: "please adjust in embedded Milvus: local, available values are [local, remote, opendal], value minio is deprecated, use remote instead",
|
||||
Doc: "please adjust in embedded Milvus: local, available values are [local, remote], value minio is deprecated, use remote instead",
|
||||
Export: true,
|
||||
}
|
||||
p.StorageType.Init(base.mgr)
|
||||
|
||||
@@ -23,15 +23,14 @@ if [[ ${SKIP_3RDPARTY} -eq 1 ]]; then
|
||||
fi
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-o BUILD_OPENDAL] [-t BUILD_TYPE] [-h]"
|
||||
echo " -o BUILD_OPENDAL Enable/disable OpenDAL build (ON/OFF, default: OFF)"
|
||||
echo "Usage: $0 [-t BUILD_TYPE] [-h]"
|
||||
echo " -t BUILD_TYPE Set build type (Debug/Release/RelWithDebInfo/MinSizeRel, default: Release)"
|
||||
echo " -h Show this help message"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Build with default settings (Release, OpenDAL OFF)"
|
||||
echo " $0 # Build with default settings (Release)"
|
||||
echo " $0 -t Debug # Build in Debug mode"
|
||||
echo " $0 -o ON -t RelWithDebInfo # Build with OpenDAL enabled and RelWithDebInfo"
|
||||
echo " $0 -t RelWithDebInfo # Build with RelWithDebInfo"
|
||||
}
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
@@ -41,13 +40,9 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
|
||||
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
|
||||
BUILD_OPENDAL="OFF"
|
||||
BUILD_TYPE="Release"
|
||||
while getopts "o:t:h" arg; do
|
||||
while getopts "t:h" arg; do
|
||||
case $arg in
|
||||
o)
|
||||
BUILD_OPENDAL=$OPTARG
|
||||
;;
|
||||
t)
|
||||
BUILD_TYPE=$OPTARG
|
||||
;;
|
||||
|
||||
@@ -99,7 +99,6 @@ EMBEDDED_MILVUS="OFF"
|
||||
BUILD_DISK_ANN="OFF"
|
||||
USE_ASAN="OFF"
|
||||
USE_DYNAMIC_SIMD="ON"
|
||||
USE_OPENDAL="OFF"
|
||||
USE_SVS="OFF"
|
||||
TANTIVY_FEATURES=""
|
||||
INDEX_ENGINE="KNOWHERE"
|
||||
@@ -112,7 +111,7 @@ fi
|
||||
: "${USE_PCH:="ON"}"
|
||||
: "${USE_UNITY_BUILD:="OFF"}"
|
||||
|
||||
while getopts "p:t:s:n:a:y:x:o:f:S:ulcgbZh" arg; do
|
||||
while getopts "p:t:s:n:a:y:x:f:S:ulcgbZh" arg; do
|
||||
case $arg in
|
||||
p)
|
||||
INSTALL_PREFIX=$OPTARG
|
||||
@@ -155,9 +154,6 @@ while getopts "p:t:s:n:a:y:x:o:f:S:ulcgbZh" arg; do
|
||||
x)
|
||||
INDEX_ENGINE=$OPTARG
|
||||
;;
|
||||
o)
|
||||
USE_OPENDAL=$OPTARG
|
||||
;;
|
||||
S)
|
||||
USE_SVS=$OPTARG
|
||||
;;
|
||||
@@ -180,7 +176,6 @@ parameter:
|
||||
-b: build embedded milvus(default: OFF)
|
||||
-a: build milvus with AddressSanitizer(default: false)
|
||||
-Z: build milvus without azure-sdk-for-cpp, so cannot use azure blob
|
||||
-o: build milvus with opendal(default: false)
|
||||
-S: build milvus with SVS/Intel Scalable Vector Search(default: OFF)
|
||||
-f: build milvus with tantivy features(default: '')
|
||||
-h: help
|
||||
@@ -254,7 +249,6 @@ ${CMAKE_EXTRA_ARGS} \
|
||||
-DUSE_ASAN=${USE_ASAN} \
|
||||
-DUSE_DYNAMIC_SIMD=${USE_DYNAMIC_SIMD} \
|
||||
-DCPU_ARCH=${CPU_ARCH} \
|
||||
-DUSE_OPENDAL=${USE_OPENDAL} \
|
||||
-DWITH_SVS=${USE_SVS} \
|
||||
-DINDEX_ENGINE=${INDEX_ENGINE} \
|
||||
-DTANTIVY_FEATURES_LIST=${TANTIVY_FEATURES} \
|
||||
|
||||
Reference in New Issue
Block a user