furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance: remove kScalarIndexUseV3 global flag and migrate UT to V3 APIs (#48839)

Browse Source 
- Remove kScalarIndexUseV3 from Meta.h/Meta.cpp and init_gtest.cpp
- Remove if(kScalarIndexUseV3) branches from 8 index source files,
keeping only the V2 Upload/Load code paths for production use
- Migrate all C++ UT scalar index calls from Upload/Load to
UploadV3/LoadV3 (44 call sites across 7 test files)
- Remove V2-only compat test (ScalarIndexV2Compat.ForceV2Upload)
- Clean up RTreeIndexTest V2 dead code paths
- Remove kScalarIndexUseV3 ternary from FileManager.h
GetRemoteIndexObjectPrefix/GetRemoteTextLogPrefix

issue: https://github.com/milvus-io/milvus/issues/47417

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
Buqian Zheng
2026-04-09 11:19:38 +08:00
committed by GitHub
parent d3ec9ee750
commit a1bb66ebfe
24 changed files with 215 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/core/src/exec/expression/LikeConjunctExprTest.cpp
+2 -1
View File
@@ -227,7 +227,7 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) {
auto index =
std::make_shared<index::NgramInvertedIndex>(ctx, ngram_params);
index->Build(config);
auto create_index_result = index->Upload();
auto create_index_result = index->UploadV3({});
auto index_files = create_index_result->GetIndexFiles();
std::map<std::string, std::string> index_params{
@@ -235,6 +235,7 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) {
{milvus::index::MIN_GRAM, "2"},
{milvus::index::MAX_GRAM, "4"},
{milvus::LOAD_PRIORITY, "HIGH"},
{milvus::index::SCALAR_INDEX_ENGINE_VERSION, "3"},
};
LoadIndexInfo load_index_info;
load_index_info.collection_id = collection_id;
internal/core/src/index/BitmapIndex.cpp
-7
View File
@@ -291,9 +291,6 @@ BitmapIndex<T>::Serialize(const Config& config) {
template <typename T>
IndexStatsPtr
BitmapIndex<T>::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return this->UploadV3(config);
}
auto binary_set = Serialize(config);
this->file_manager_->AddFile(binary_set);
@@ -588,10 +585,6 @@ template <typename T>
void
BitmapIndex<T>::Load(milvus::tracer::TraceContext ctx, const Config& config) {
LOG_INFO("load bitmap index with config {}", config.dump());
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
AssertInfo(index_files.has_value(),
internal/core/src/index/BitmapIndexArrayTest.cpp
+3 -2
View File
@@ -231,6 +231,7 @@ class ArrayBitmapIndexTest : public testing::Test {
config[INSERT_FILES_KEY] = std::vector<std::string>{log_path};
config["bitmap_cardinality_limit"] = "100";
config[INDEX_NUM_ROWS_KEY] = nb_;
config[milvus::index::SCALAR_INDEX_ENGINE_VERSION] = 3;
if (has_lack_binlog_row_) {
config[INDEX_NUM_ROWS_KEY] = nb_ + lack_binlog_row_;
}
@@ -259,7 +260,7 @@ class ArrayBitmapIndexTest : public testing::Test {
ctx.set_for_loading_index(true);
index_ =
index::IndexFactory::GetInstance().CreateIndex(index_info, ctx);
index_->Load(milvus::tracer::TraceContext{}, config);
index_->LoadV3(config);
}
virtual void
@@ -482,4 +483,4 @@ INSTANTIATE_TYPED_TEST_SUITE_P(ArrayBitmapE2ECheckV1,
INSTANTIATE_TYPED_TEST_SUITE_P(ArrayBitmapE2ECheckV1,
ArrayBitmapIndexTestV2,
BitmapTypeV1);
BitmapTypeV1);
internal/core/src/index/BitmapIndexTest.cpp
+3 -2
View File
@@ -176,6 +176,7 @@ class BitmapIndexTest : public testing::Test {
config["index_type"] = milvus::index::BITMAP_INDEX_TYPE;
config[INSERT_FILES_KEY] = std::vector<std::string>{log_path};
config[INDEX_NUM_ROWS_KEY] = nb_;
config[milvus::index::SCALAR_INDEX_ENGINE_VERSION] = 3;
if (has_lack_binlog_row_) {
config[INDEX_NUM_ROWS_KEY] = nb_ + lack_binlog_row_;
}
@@ -212,7 +213,7 @@ class BitmapIndexTest : public testing::Test {
milvus::proto::common::LoadPriority::HIGH;
index_ =
index::IndexFactory::GetInstance().CreateIndex(index_info, ctx);
index_->Load(milvus::tracer::TraceContext{}, config);
index_->LoadV3(config);
}
virtual void
@@ -912,4 +913,4 @@ REGISTER_TYPED_TEST_SUITE_P(BitmapIndexTestV6,
INSTANTIATE_TYPED_TEST_SUITE_P(BitmapIndexE2ECheck_Mmap,
BitmapIndexTestV6,
BitmapType);
BitmapType);
internal/core/src/index/BoolIndexTest.cpp
+55 -9
View File
@@ -21,9 +21,34 @@
#include "common/protobuf_utils.h"
#include "gtest/gtest.h"
#include "index/ScalarIndexSort.h"
#include "pb/common.pb.h"
#include "storage/ChunkManager.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/AssertUtils.h"
#include "test_utils/Constants.h"
#include "test_utils/indexbuilder_test_utils.h"
namespace {
milvus::storage::FileManagerContext
CreateBoolTestFileManagerContext() {
milvus::storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = TestLocalPath;
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
milvus::storage::FieldDataMeta field_meta{1, 2, 3, 101};
field_meta.field_schema.set_data_type(
milvus::proto::schema::DataType::Bool);
milvus::storage::IndexMeta index_meta{3, 101, 1000, 10000};
milvus::storage::FileManagerContext ctx(
field_meta, index_meta, chunk_manager, fs);
return ctx;
}
} // namespace
class BoolIndexTest : public ::testing::Test {
protected:
void
@@ -171,11 +196,18 @@ TEST_F(BoolIndexTest, Codec) {
auto false_test = std::make_unique<bool>(false);
{
auto index = milvus::index::CreateBoolIndex();
auto file_manager_ctx = CreateBoolTestFileManagerContext();
auto index = milvus::index::CreateBoolIndex(file_manager_ctx);
index->Build(all_true.data_size(), all_true.data().data());
auto copy_index = milvus::index::CreateBoolIndex();
copy_index->Load(index->Serialize(nullptr));
auto copy_index = milvus::index::CreateBoolIndex(file_manager_ctx);
auto create_index_result = index->UploadV3({});
auto index_files = create_index_result->GetIndexFiles();
milvus::Config load_config;
load_config["index_files"] = index_files;
load_config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
copy_index->LoadV3(load_config);
auto bitset1 = copy_index->NotIn(1, true_test.get());
ASSERT_TRUE(bitset1.none());
@@ -185,11 +217,18 @@ TEST_F(BoolIndexTest, Codec) {
}
{
auto index = milvus::index::CreateBoolIndex();
auto file_manager_ctx = CreateBoolTestFileManagerContext();
auto index = milvus::index::CreateBoolIndex(file_manager_ctx);
index->Build(all_false.data_size(), all_false.data().data());
auto copy_index = milvus::index::CreateBoolIndex();
copy_index->Load(index->Serialize(nullptr));
auto copy_index = milvus::index::CreateBoolIndex(file_manager_ctx);
auto create_index_result = index->UploadV3({});
auto index_files = create_index_result->GetIndexFiles();
milvus::Config load_config;
load_config["index_files"] = index_files;
load_config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
copy_index->LoadV3(load_config);
auto bitset1 = copy_index->NotIn(1, true_test.get());
ASSERT_TRUE(bitset1.any());
@@ -199,11 +238,18 @@ TEST_F(BoolIndexTest, Codec) {
}
{
auto index = milvus::index::CreateBoolIndex();
auto file_manager_ctx = CreateBoolTestFileManagerContext();
auto index = milvus::index::CreateBoolIndex(file_manager_ctx);
index->Build(half.data_size(), half.data().data());
auto copy_index = milvus::index::CreateBoolIndex();
copy_index->Load(index->Serialize(nullptr));
auto copy_index = milvus::index::CreateBoolIndex(file_manager_ctx);
auto create_index_result = index->UploadV3({});
auto index_files = create_index_result->GetIndexFiles();
milvus::Config load_config;
load_config["index_files"] = index_files;
load_config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
copy_index->LoadV3(load_config);
auto bitset1 = copy_index->NotIn(1, true_test.get());
for (size_t i = 0; i < n; i++) {
internal/core/src/index/HybridScalarIndex.cpp
-7
View File
@@ -319,9 +319,6 @@ HybridScalarIndex<T>::SerializeIndexType() {
template <typename T>
IndexStatsPtr
HybridScalarIndex<T>::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return this->UploadV3(config);
}
auto internal_index = GetInternalIndex();
auto index_ret = internal_index->Upload(config);
@@ -376,10 +373,6 @@ template <typename T>
void
HybridScalarIndex<T>::Load(milvus::tracer::TraceContext ctx,
const Config& config) {
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
AssertInfo(index_files.has_value(),
internal/core/src/index/HybridScalarIndexTest.cpp
+2 -1
View File
@@ -178,6 +178,7 @@ class HybridIndexTestV1 : public testing::Test {
config[INSERT_FILES_KEY] = std::vector<std::string>{log_path};
config["bitmap_cardinality_limit"] = "1000";
config[INDEX_NUM_ROWS_KEY] = nb_;
config[milvus::index::SCALAR_INDEX_ENGINE_VERSION] = 3;
if (has_lack_binlog_row_) {
config[INDEX_NUM_ROWS_KEY] = nb_ + lack_binlog_row_;
}
@@ -206,7 +207,7 @@ class HybridIndexTestV1 : public testing::Test {
ctx.set_for_loading_index(true);
index_ =
index::IndexFactory::GetInstance().CreateIndex(index_info, ctx);
index_->Load(milvus::tracer::TraceContext{}, config);
index_->LoadV3(config);
}
virtual void
internal/core/src/index/InvertedIndexTantivy.cpp
-7
View File
@@ -155,9 +155,6 @@ InvertedIndexTantivy<T>::Serialize(const Config& config) {
template <typename T>
IndexStatsPtr
InvertedIndexTantivy<T>::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return this->UploadV3(config);
}
finish();
boost::filesystem::path p(path_);
@@ -218,10 +215,6 @@ template <typename T>
void
InvertedIndexTantivy<T>::Load(milvus::tracer::TraceContext ctx,
const Config& config) {
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, INDEX_FILES);
AssertInfo(index_files.has_value(),
internal/core/src/index/InvertedIndexTest.cpp
+4 -2
View File
@@ -216,6 +216,7 @@ test_run() {
config["index_type"] = milvus::index::INVERTED_INDEX_TYPE;
config[INSERT_FILES_KEY] = std::vector<std::string>{log_path};
config[INDEX_NUM_ROWS_KEY] = nb;
config[milvus::index::SCALAR_INDEX_ENGINE_VERSION] = 3;
if (has_lack_binlog_row_) {
config[INDEX_NUM_ROWS_KEY] = nb + lack_binlog_row;
}
@@ -244,7 +245,7 @@ test_run() {
ctx.set_for_loading_index(true);
auto index =
index::IndexFactory::GetInstance().CreateIndex(index_info, ctx);
index->Load(milvus::tracer::TraceContext{}, config);
index->LoadV3(config);
auto cnt = index->Count();
if (has_lack_binlog_row_) {
@@ -611,6 +612,7 @@ test_string() {
config["index_type"] = milvus::index::INVERTED_INDEX_TYPE;
config[INSERT_FILES_KEY] = std::vector<std::string>{log_path};
config[INDEX_NUM_ROWS_KEY] = nb;
config[milvus::index::SCALAR_INDEX_ENGINE_VERSION] = 3;
if (has_lack_binlog_row_) {
config[INDEX_NUM_ROWS_KEY] = nb + lack_binlog_row;
}
@@ -639,7 +641,7 @@ test_string() {
ctx.set_for_loading_index(true);
auto index =
index::IndexFactory::GetInstance().CreateIndex(index_info, ctx);
index->Load(milvus::tracer::TraceContext{}, config);
index->LoadV3(config);
auto cnt = index->Count();
if (has_lack_binlog_row_) {
internal/core/src/index/JsonFlatIndexTest.cpp
+2 -2
View File
@@ -165,7 +165,7 @@ class JsonFlatIndexTest : public ::testing::Test {
auto index = std::make_shared<index::JsonFlatIndex>(*ctx_, "");
index->Build(config);
auto create_index_result = index->Upload();
auto create_index_result = index->UploadV3({});
auto memSize = create_index_result->GetMemSize();
auto serializedSize = create_index_result->GetSerializedSize();
ASSERT_GT(memSize, 0);
@@ -185,7 +185,7 @@ class JsonFlatIndexTest : public ::testing::Test {
ctx_->set_for_loading_index(true);
json_index_ = std::make_shared<index::JsonFlatIndex>(*ctx_, "");
json_index_->Load(milvus::tracer::TraceContext{}, load_config);
json_index_->LoadV3(load_config);
auto cnt = json_index_->Count();
ASSERT_EQ(cnt, json_data_.size());
internal/core/src/index/Meta.cpp
+1 -1
View File
@@ -17,5 +17,5 @@
#include "index/Meta.h"
namespace milvus::index {
bool kScalarIndexUseV3 = false;
std::string kOverrideRootPathForUT;
} // namespace milvus::index
internal/core/src/index/Meta.h
+5 -4
View File
@@ -112,8 +112,9 @@ constexpr const char* DISK_ANN_PREPARE_USE_BFS_CACHE = "use_bfs_cache";
// DiskAnn query params
constexpr const char* DISK_ANN_QUERY_LIST = "search_list";
constexpr const char* DISK_ANN_QUERY_BEAMWIDTH = "beamwidth";
// UT-only flag: when true, index Upload()/Load() route to V3 paths.
// In production, V2/V3 routing is handled by callers (ScalarIndexCreator, SealedIndexTranslator).
// Delete this along with V2 Upload/Load code when V2 compatibility is dropped.
extern bool kScalarIndexUseV3;
// UT-only: when non-empty, overrides rcm_->GetRootPath() in
// GetRemoteIndexObjectPrefix/GetRemoteTextLogPrefix to avoid
// absolute path duplication with SubTreeFileSystem in tests.
extern std::string kOverrideRootPathForUT;
} // namespace milvus::index
internal/core/src/index/NgramInvertedIndex.cpp
-7
View File
@@ -207,9 +207,6 @@ NgramInvertedIndex::Serialize(const Config& config) {
IndexStatsPtr
NgramInvertedIndex::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return UploadV3(config);
}
finish();
auto index_build_end = std::chrono::system_clock::now();
auto index_build_duration =
@@ -301,10 +298,6 @@ NgramInvertedIndex::RetainTantivyIndexFiles(
void
NgramInvertedIndex::Load(milvus::tracer::TraceContext ctx,
const Config& config) {
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, INDEX_FILES);
AssertInfo(index_files.has_value(),
internal/core/src/index/NgramInvertedIndexTest.cpp
+10 -8
View File
@@ -177,7 +177,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
std::make_shared<index::NgramInvertedIndex>(ctx, ngram_params);
index->Build(config);
auto create_index_result = index->Upload();
auto create_index_result = index->UploadV3({});
auto memSize = create_index_result->GetMemSize();
index_size = create_index_result->GetSerializedSize();
ASSERT_GT(memSize, 0);
@@ -194,7 +194,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
auto ngram_params = index::NgramParams{true, 2, 4};
auto index =
std::make_unique<index::NgramInvertedIndex>(ctx, ngram_params);
index->Load(milvus::tracer::TraceContext{}, config);
index->LoadV3(config);
auto cnt = index->Count();
ASSERT_EQ(cnt, nb);
@@ -229,6 +229,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
{milvus::index::MIN_GRAM, "2"},
{milvus::index::MAX_GRAM, "4"},
{milvus::LOAD_PRIORITY, "HIGH"},
{milvus::index::SCALAR_INDEX_ENGINE_VERSION, "3"},
};
milvus::segcore::LoadIndexInfo load_index_info{};
load_index_info.collection_id = collection_id;
@@ -500,7 +501,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
std::make_shared<index::NgramInvertedIndex>(ctx, ngram_params);
index->Build(config);
auto create_index_result = index->Upload();
auto create_index_result = index->UploadV3({});
index_files = create_index_result->GetIndexFiles();
}
@@ -511,6 +512,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
{milvus::index::MIN_GRAM, "2"},
{milvus::index::MAX_GRAM, "4"},
{milvus::LOAD_PRIORITY, "HIGH"},
{milvus::index::SCALAR_INDEX_ENGINE_VERSION, "3"},
};
milvus::segcore::LoadIndexInfo load_index_info{};
load_index_info.collection_id = collection_id;
@@ -1652,7 +1654,7 @@ TEST(NgramBenchmark, NgramVsTantivyVsBruteForce) {
auto index =
std::make_shared<index::NgramInvertedIndex>(ctx, ngram_params);
index->Build(config);
auto result = index->Upload();
auto result = index->UploadV3({});
index_files = result->GetIndexFiles();
}
@@ -1665,7 +1667,7 @@ TEST(NgramBenchmark, NgramVsTantivyVsBruteForce) {
index::NgramParams{.loading_index = true, .min_gram = 2, .max_gram = 4};
auto ngram_index =
std::make_unique<index::NgramInvertedIndex>(ctx, load_ngram_params);
ngram_index->Load(milvus::tracer::TraceContext{}, load_config);
ngram_index->LoadV3(load_config);
// Build Tantivy index for comparison
std::vector<std::string> data_vec(data.begin(), data.end());
@@ -1933,7 +1935,7 @@ TEST(NgramBenchmark, NgramFilteringEffectiveness) {
auto index =
std::make_shared<index::NgramInvertedIndex>(ctx, ngram_params);
index->Build(config);
auto result = index->Upload();
auto result = index->UploadV3({});
index_files = result->GetIndexFiles();
}
@@ -1945,7 +1947,7 @@ TEST(NgramBenchmark, NgramFilteringEffectiveness) {
index::NgramParams{.loading_index = true, .min_gram = 2, .max_gram = 4};
auto ngram_index =
std::make_unique<index::NgramInvertedIndex>(ctx, load_ngram_params);
ngram_index->Load(milvus::tracer::TraceContext{}, load_config);
ngram_index->LoadV3(load_config);
std::vector<std::string> data_vec(data.begin(), data.end());
@@ -2145,4 +2147,4 @@ TEST(NgramBenchmark, NgramFilteringEffectiveness) {
<< " " << fc.reason << "\n";
}
}
}
}
internal/core/src/index/RTreeIndex.cpp
-7
View File
@@ -129,10 +129,6 @@ template <typename T>
void
RTreeIndex<T>::Load(milvus::tracer::TraceContext ctx, const Config& config) {
LOG_DEBUG("Load RTreeIndex with config {}", config.dump());
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files_opt =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
@@ -338,9 +334,6 @@ RTreeIndex<T>::finish() {
template <typename T>
IndexStatsPtr
RTreeIndex<T>::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return this->UploadV3(config);
}
// 1. Ensure all buffered data flushed to disk
finish();
internal/core/src/index/RTreeIndexTest.cpp
+37 -91
View File
@@ -223,7 +223,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load) {
ASSERT_EQ(rtree_build.Count(), 2);
// ---------- Upload ----------
auto stats = rtree_build.Upload({});
auto stats = rtree_build.UploadV3({});
ASSERT_NE(stats, nullptr);
ASSERT_GT(stats->GetIndexFiles().size(), 0);
@@ -237,7 +237,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load) {
cfg["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx; // empty context
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
ASSERT_EQ(rtree_load.Count(), 2);
}
@@ -252,20 +252,15 @@ TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) {
CreatePointWKB(20.0, 20.0)};
rtree_build.BuildWithRawDataForUT(wkbs2.size(), wkbs2.data());
auto stats = rtree_build.Upload({});
auto stats = rtree_build.UploadV3({});
// gather only filenames (strip parent path)
std::vector<std::string> filenames;
for (const auto& path : stats->GetIndexFiles()) {
filenames.emplace_back(
boost::filesystem::path(path).filename().string());
// In V2 mode, files are stored via chunk_manager.
// In V3 mode, files are stored via ArrowFileSystem (fs_),
// V3 mode: files are stored via ArrowFileSystem (fs_),
// so chunk_manager won't find them.
if (!milvus::index::kScalarIndexUseV3) {
ASSERT_TRUE(chunk_manager_->Exist(path));
ASSERT_GT(chunk_manager_->Size(path), 0);
}
}
// Load using filename only list
@@ -278,7 +273,7 @@ TEST_F(RTreeIndexTest, Load_WithFileNamesOnly) {
cfg["index_files"] = filenames; // no directory info
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
ASSERT_EQ(rtree_load.Count(), 2);
}
@@ -306,7 +301,7 @@ TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) {
rtree.BuildWithRawDataForUT(wkbs.size(), wkbs.data());
// Upload and then load back to let loader compute count from wrapper
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_, fs_);
@@ -316,7 +311,7 @@ TEST_F(RTreeIndexTest, Build_WithInvalidWKB_Upload_Load) {
nlohmann::json cfg;
cfg["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
// Only 2 valid points should be present
ASSERT_EQ(rtree_load.Count(), 2);
@@ -337,7 +332,7 @@ TEST_F(RTreeIndexTest, Build_VariousGeometries) {
rtree.BuildWithRawDataForUT(wkbs.size(), wkbs.data());
ASSERT_EQ(rtree.Count(), wkbs.size());
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
ASSERT_FALSE(stats->GetIndexFiles().empty());
milvus::storage::FileManagerContext ctx_load(
@@ -348,7 +343,7 @@ TEST_F(RTreeIndexTest, Build_VariousGeometries) {
nlohmann::json cfg;
cfg["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
ASSERT_EQ(rtree_load.Count(), wkbs.size());
}
@@ -366,71 +361,22 @@ TEST_F(RTreeIndexTest, Build_ConfigAndMetaJson) {
build_cfg["insert_files"] = std::vector<std::string>{remote_file};
rtree.Build(build_cfg);
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
if (milvus::index::kScalarIndexUseV3) {
// V3 mode: verify upload produced a single packed file and can be loaded
auto index_files = stats->GetIndexFiles();
ASSERT_EQ(index_files.size(), 1);
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
nlohmann::json cfg;
cfg["index_files"] = index_files;
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
ASSERT_EQ(rtree_load.Count(), 2);
return;
}
// Cache remote index files locally
milvus::storage::DiskFileManagerImpl diskfm(
{field_meta_, index_meta_, chunk_manager_, fs_});
// V3 mode: verify upload produced a single packed file and can be loaded
auto index_files = stats->GetIndexFiles();
auto load_priority =
milvus::index::GetValueFromConfig<milvus::proto::common::LoadPriority>(
build_cfg, milvus::LOAD_PRIORITY)
.value_or(milvus::proto::common::LoadPriority::HIGH);
diskfm.CacheIndexToDisk(index_files, load_priority);
auto local_paths = diskfm.GetLocalFilePaths();
ASSERT_FALSE(local_paths.empty());
// Determine base path like RTreeIndex::Load
auto ends_with = [](const std::string& value, const std::string& suffix) {
return value.size() >= suffix.size() &&
value.compare(
value.size() - suffix.size(), suffix.size(), suffix) == 0;
};
ASSERT_EQ(index_files.size(), 1);
std::string base_path;
for (const auto& p : local_paths) {
if (ends_with(p, ".bgi")) {
base_path = p.substr(0, p.size() - 4);
break;
}
}
if (base_path.empty()) {
for (const auto& p : local_paths) {
if (ends_with(p, ".meta.json")) {
base_path =
p.substr(0, p.size() - std::string(".meta.json").size());
break;
}
}
}
if (base_path.empty()) {
base_path = local_paths.front();
}
// Parse local meta json
std::ifstream ifs(base_path + ".meta.json");
ASSERT_TRUE(ifs.good());
nlohmann::json meta = nlohmann::json::parse(ifs);
ASSERT_EQ(meta["dimension"], 2);
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
milvus::index::RTreeIndex<std::string> rtree_load(ctx_load);
// Clean up config and meta test files
CleanupIndexFiles(stats->GetIndexFiles(), "config test");
nlohmann::json cfg;
cfg["index_files"] = index_files;
milvus::tracer::TraceContext trace_ctx;
rtree_load.LoadV3(cfg);
ASSERT_EQ(rtree_load.Count(), 2);
}
TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) {
@@ -441,7 +387,7 @@ TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) {
std::vector<std::string> wkbs = {CreatePointWKB(6.0, 6.0),
CreatePointWKB(7.0, 7.0)};
rtree.BuildWithRawDataForUT(wkbs.size(), wkbs.data());
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
// Use full list, but replace one with filename-only
auto mixed = stats->GetIndexFiles();
@@ -456,7 +402,7 @@ TEST_F(RTreeIndexTest, Load_MixedFileNamesAndPaths) {
nlohmann::json cfg;
cfg["index_files"] = mixed;
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
ASSERT_EQ(rtree_load.Count(), wkbs.size());
}
@@ -471,7 +417,7 @@ TEST_F(RTreeIndexTest, Load_NonexistentRemote_ShouldThrow) {
cfg["index_files"] = std::vector<std::string>{
(temp_path_.get() / "does_not_exist.bgi_0").string()};
milvus::tracer::TraceContext trace_ctx;
EXPECT_THROW(rtree_load.Load(trace_ctx, cfg), milvus::SegcoreError);
EXPECT_THROW(rtree_load.LoadV3(cfg), milvus::SegcoreError);
}
TEST_F(RTreeIndexTest, Build_EndToEnd_FromInsertFiles) {
@@ -491,7 +437,7 @@ TEST_F(RTreeIndexTest, Build_EndToEnd_FromInsertFiles) {
rtree.Build(build_cfg);
ASSERT_EQ(rtree.Count(), wkbs.size());
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_, fs_);
@@ -500,7 +446,7 @@ TEST_F(RTreeIndexTest, Build_EndToEnd_FromInsertFiles) {
nlohmann::json cfg;
cfg["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
ASSERT_EQ(rtree_load.Count(), wkbs.size());
}
@@ -532,7 +478,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load_LargeDataset) {
ASSERT_EQ(rtree.Count(), static_cast<int64_t>(N));
// Upload index
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
ASSERT_GT(stats->GetIndexFiles().size(), 0);
// Load index back and verify
@@ -544,7 +490,7 @@ TEST_F(RTreeIndexTest, Build_Upload_Load_LargeDataset) {
nlohmann::json cfg_load;
cfg_load["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg_load);
rtree_load.LoadV3(cfg_load);
ASSERT_EQ(rtree_load.Count(), static_cast<int64_t>(N));
@@ -587,7 +533,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) {
ASSERT_EQ(rtree.Count(), 4);
// upload -> load back and verify consistency
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
ASSERT_GT(stats->GetIndexFiles().size(), 0);
milvus::storage::FileManagerContext ctx_load(
@@ -599,7 +545,7 @@ TEST_F(RTreeIndexTest, Build_BulkLoad_Nulls_And_BadWKB) {
cfg["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
ASSERT_EQ(rtree_load.Count(), 4);
}
@@ -622,7 +568,7 @@ TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) {
ASSERT_EQ(rtree.Count(), 3);
// Upload and then load into a new index instance for querying
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
@@ -630,7 +576,7 @@ TEST_F(RTreeIndexTest, Query_CoarseAndExact_Equals_Intersects_Within) {
nlohmann::json cfg;
cfg["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
// Helper to run Query
auto run_query = [&](::milvus::proto::plan::GISFunctionFilterExpr_GISOp op,
@@ -698,7 +644,7 @@ TEST_F(RTreeIndexTest, Query_Touches_Contains_Crosses_Overlaps) {
ASSERT_EQ(rtree.Count(), 3);
// Upload and load a new instance for querying
auto stats = rtree.Upload({});
auto stats = rtree.UploadV3({});
milvus::storage::FileManagerContext ctx_load(
field_meta_, index_meta_, chunk_manager_, fs_);
ctx_load.set_for_loading_index(true);
@@ -706,7 +652,7 @@ TEST_F(RTreeIndexTest, Query_Touches_Contains_Crosses_Overlaps) {
nlohmann::json cfg;
cfg["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx;
rtree_load.Load(trace_ctx, cfg);
rtree_load.LoadV3(cfg);
auto run_query = [&](::milvus::proto::plan::GISFunctionFilterExpr_GISOp op,
const std::string& wkt) {
@@ -830,7 +776,7 @@ TEST_F(RTreeIndexTest, GIS_Index_Exact_Filtering) {
build_cfg["index_type"] = milvus::index::RTREE_INDEX_TYPE;
rtree_index->Build(build_cfg);
auto stats = rtree_index->Upload({});
auto stats = rtree_index->UploadV3({});
// load geometry index into sealed segment
milvus::segcore::LoadIndexInfo info{};
@@ -849,7 +795,7 @@ TEST_F(RTreeIndexTest, GIS_Index_Exact_Filtering) {
nlohmann::json cfg_load;
cfg_load["index_files"] = stats->GetIndexFiles();
milvus::tracer::TraceContext trace_ctx_load;
rtree_index->Load(trace_ctx_load, cfg_load);
rtree_index->LoadV3(cfg_load);
info.cache_index =
CreateTestCacheIndex("rtree_index_key", std::move(rtree_index));
@@ -891,4 +837,4 @@ TEST_F(RTreeIndexTest, GIS_Index_Exact_Filtering) {
// Clean up any remaining index files
CleanupIndexFiles(stats->GetIndexFiles(), "GIS filtering test");
}
}
internal/core/src/index/ScalarIndexSort.cpp
-7
View File
@@ -260,9 +260,6 @@ ScalarIndexSort<T>::Serialize(const Config& config) {
template <typename T>
IndexStatsPtr
ScalarIndexSort<T>::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return this->UploadV3(config);
}
auto index_build_duration =
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now() - index_build_begin_)
@@ -408,10 +405,6 @@ template <typename T>
void
ScalarIndexSort<T>::Load(milvus::tracer::TraceContext ctx,
const Config& config) {
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
AssertInfo(index_files.has_value(),
internal/core/src/index/ScalarIndexSortTest.cpp
+27 -114
View File
@@ -20,13 +20,29 @@
#include "storage/ChunkManager.h"
#include "storage/FileManager.h"
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/TmpPath.h"
#include "test_utils/storage_test_utils.h"
using namespace milvus;
using namespace milvus::index;
static storage::FileManagerContext
CreateScalarSortTestFileManagerContext() {
storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = TestLocalPath;
auto chunk_manager = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
storage::FieldDataMeta field_meta{1, 2, 3, 101};
field_meta.field_schema.set_data_type(proto::schema::DataType::Int64);
storage::IndexMeta index_meta{3, 101, 1000, 10000};
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager, fs);
return ctx;
}
void
test_stlsort_for_range(
const std::vector<int64_t>& data,
@@ -36,23 +52,27 @@ test_stlsort_for_range(
const std::shared_ptr<ScalarIndexSort<int64_t>>&)> exec_expr,
const std::vector<bool>& expected_result) {
size_t nb = data.size();
BinarySet binary_set;
std::vector<std::string> index_files;
{
Config config;
auto index = std::make_shared<index::ScalarIndexSort<int64_t>>();
auto index = std::make_shared<index::ScalarIndexSort<int64_t>>(
CreateScalarSortTestFileManagerContext());
index->Build(nb, data.data());
binary_set = index->Serialize(config);
auto create_index_result = index->UploadV3({});
index_files = create_index_result->GetIndexFiles();
}
{
Config config;
config[milvus::index::ENABLE_MMAP] = enable_mmap;
config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
config["index_files"] = index_files;
auto index = std::make_shared<index::ScalarIndexSort<int64_t>>();
index->Load(binary_set, config);
auto index = std::make_shared<index::ScalarIndexSort<int64_t>>(
CreateScalarSortTestFileManagerContext());
index->LoadV3(config);
auto cnt = index->Count();
ASSERT_EQ(cnt, nb);
@@ -112,112 +132,5 @@ TEST(StlSortIndexTest, TestIn) {
data, DataType::INT64, true, exec_expr, expected_result);
}
// Verify that forcing kScalarIndexUseV3=false still produces valid V2
// index files (multiple files) and that the data round-trips correctly.
TEST(ScalarIndexV2Compat, ForceV2Upload) {
// RAII guard to save and restore kScalarIndexUseV3
const bool original_value = milvus::index::kScalarIndexUseV3;
struct VersionGuard {
bool saved;
~VersionGuard() {
milvus::index::kScalarIndexUseV3 = saved;
}
} guard{original_value};
// Force V2 mode
milvus::index::kScalarIndexUseV3 = false;
// Setup local storage and ArrowFileSystem via TmpPath
milvus::test::TmpPath temp_path;
auto storage_config = gen_local_storage_config(temp_path.get().string());
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
// Prepare field and index metadata
auto field_meta = milvus::segcore::gen_field_meta(
/*collection_id=*/1,
/*partition_id=*/1,
/*segment_id=*/1,
/*field_id=*/100,
DataType::INT64);
auto index_meta = gen_index_meta(
/*segment_id=*/1,
/*field_id=*/100,
/*index_build_id=*/1,
/*index_version=*/1);
// Create FileManagerContext for building
milvus::storage::FileManagerContext ctx_build(
field_meta, index_meta, chunk_manager, fs);
// Build index with test data
const size_t nb = 100;
std::vector<int64_t> data(nb);
for (size_t i = 0; i < nb; ++i) {
data[i] = static_cast<int64_t>(nb - i); // reverse order: 100..1
}
auto index_build =
std::make_shared<milvus::index::ScalarIndexSort<int64_t>>(ctx_build);
index_build->Build(nb, data.data());
ASSERT_EQ(index_build->Count(), static_cast<int64_t>(nb));
// Upload (should use V2 path since kScalarIndexUseV3 == false)
auto stats = index_build->Upload({});
ASSERT_NE(stats, nullptr);
// V2 format produces multiple files (index_data, index_length, etc.)
auto index_files = stats->GetIndexFiles();
ASSERT_GT(index_files.size(), 1)
<< "V2 upload should produce multiple files, got "
<< index_files.size();
// Verify the files exist in chunk manager
for (const auto& file : index_files) {
ASSERT_TRUE(chunk_manager->Exist(file))
<< "Index file should exist: " << file;
ASSERT_GT(chunk_manager->Size(file), 0)
<< "Index file should be non-empty: " << file;
}
// Load back the index using the V2 Load path
milvus::storage::FileManagerContext ctx_load(
field_meta, index_meta, chunk_manager, fs);
auto index_load =
std::make_shared<milvus::index::ScalarIndexSort<int64_t>>(ctx_load);
nlohmann::json load_config;
load_config[milvus::index::INDEX_FILES] = index_files;
load_config[milvus::index::ENABLE_MMAP] = false;
load_config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
milvus::tracer::TraceContext trace_ctx;
index_load->Load(trace_ctx, load_config);
ASSERT_EQ(index_load->Count(), static_cast<int64_t>(nb));
// Verify data correctness via Reverse_Lookup
for (size_t i = 0; i < nb; ++i) {
auto val = index_load->Reverse_Lookup(i);
ASSERT_TRUE(val.has_value())
<< "Reverse_Lookup should succeed at " << i;
ASSERT_EQ(val.value(), data[i]) << "Value mismatch at offset " << i;
}
// Verify query correctness: Range [50, 60] inclusive
auto bitset = index_load->Range(
static_cast<int64_t>(50), true, static_cast<int64_t>(60), true);
int64_t count = 0;
for (size_t i = 0; i < nb; ++i) {
if (data[i] >= 50 && data[i] <= 60) {
ASSERT_TRUE(bitset[i])
<< "Expected bit set for value " << data[i] << " at " << i;
count++;
} else {
ASSERT_FALSE(bitset[i])
<< "Expected bit unset for value " << data[i] << " at " << i;
}
}
ASSERT_EQ(count, 11); // values 50..60 inclusive = 11 values
}
// V2 compat test removed: kScalarIndexUseV3 flag deleted,
// Upload()/Load() now always route to V3 paths.
internal/core/src/index/ScalarIndexTest.cpp
+17 -4
View File
@@ -39,6 +39,10 @@
#include "index/IndexInfo.h"
#include "index/ScalarIndex.h"
#include "index/ScalarIndexSort.h"
#include "pb/common.pb.h"
#include "storage/ChunkManager.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "index/StringIndexMarisa.h"
#include "pb/index_cgo_msg.pb.h"
#include "pb/schema.pb.h"
@@ -82,9 +86,13 @@ GetTempFileManagerCtx(CDataType data_type) {
storage_config.storage_type = "local";
storage_config.root_path = TestLocalPath;
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config);
auto ctx = milvus::storage::FileManagerContext(chunk_manager);
ctx.fieldDataMeta.field_schema.set_data_type(
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
milvus::storage::FieldDataMeta field_meta{1, 2, 3, 101};
field_meta.field_schema.set_data_type(
static_cast<milvus::proto::schema::DataType>(data_type));
milvus::storage::IndexMeta index_meta{3, 101, 1000, 10000};
auto ctx = milvus::storage::FileManagerContext(
field_meta, index_meta, chunk_manager, fs);
return ctx;
}
@@ -233,11 +241,16 @@ TYPED_TEST_P(TypedScalarIndexTest, Codec) {
auto arr = GenSortedArr<T>(nb);
scalar_index->Build(nb, arr.data());
auto binary_set = index->Serialize(nullptr);
auto create_index_result = index->UploadV3({});
auto index_files = create_index_result->GetIndexFiles();
auto copy_index =
milvus::index::IndexFactory::GetInstance().CreateScalarIndex(
create_index_info, GetTempFileManagerCtx(dtype));
copy_index->Load(binary_set);
milvus::Config load_config;
load_config["index_files"] = index_files;
load_config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
copy_index->LoadV3(load_config);
auto copy_scalar_index =
dynamic_cast<milvus::index::ScalarIndex<T>*>(copy_index.get());
internal/core/src/index/StringIndexMarisa.cpp
-7
View File
@@ -240,9 +240,6 @@ StringIndexMarisa::Serialize(const Config& config) {
IndexStatsPtr
StringIndexMarisa::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return UploadV3(config);
}
auto binary_set = Serialize(config);
this->file_manager_->AddFile(binary_set);
@@ -306,10 +303,6 @@ StringIndexMarisa::Load(const BinarySet& set, const Config& config) {
void
StringIndexMarisa::Load(milvus::tracer::TraceContext ctx,
const Config& config) {
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
AssertInfo(index_files.has_value(),
internal/core/src/index/StringIndexSort.cpp
-7
View File
@@ -281,9 +281,6 @@ StringIndexSort::Serialize(const Config& config) {
IndexStatsPtr
StringIndexSort::Upload(const Config& config) {
if (kScalarIndexUseV3) {
return UploadV3(config);
}
auto index_build_duration =
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now() - index_build_begin_)
@@ -309,10 +306,6 @@ StringIndexSort::Load(const BinarySet& index_binary, const Config& config) {
void
StringIndexSort::Load(milvus::tracer::TraceContext ctx, const Config& config) {
if (kScalarIndexUseV3) {
this->LoadV3(config);
return;
}
auto index_files =
GetValueFromConfig<std::vector<std::string>>(config, "index_files");
AssertInfo(index_files.has_value() && !index_files.value().empty(),
internal/core/src/index/StringIndexTest.cpp
+40 -8
View File
@@ -32,8 +32,13 @@
#include "index/StringIndex.h"
#include "index/StringIndexMarisa.h"
#include "knowhere/dataset.h"
#include "pb/common.pb.h"
#include "pb/schema.pb.h"
#include "storage/ChunkManager.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/AssertUtils.h"
#include "test_utils/Constants.h"
#include "test_utils/indexbuilder_test_utils.h"
constexpr int64_t nb = 100;
@@ -41,6 +46,21 @@ namespace schemapb = milvus::proto::schema;
namespace milvus {
namespace index {
static storage::FileManagerContext
CreateStringTestFileManagerContext() {
storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = TestLocalPath;
auto chunk_manager = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
storage::FieldDataMeta field_meta{1, 2, 3, 101};
field_meta.field_schema.set_data_type(proto::schema::DataType::VarChar);
storage::IndexMeta index_meta{3, 101, 1000, 10000};
storage::FileManagerContext ctx(field_meta, index_meta, chunk_manager, fs);
return ctx;
}
class StringIndexBaseTest : public ::testing::Test {
protected:
void
@@ -323,7 +343,8 @@ TEST_F(StringIndexMarisaTest, Query) {
}
TEST_F(StringIndexMarisaTest, Codec) {
auto index = milvus::index::CreateStringIndexMarisa();
auto file_manager_ctx = CreateStringTestFileManagerContext();
auto index = milvus::index::CreateStringIndexMarisa(file_manager_ctx);
std::vector<std::string> strings(nb);
for (int i = 0; i < nb; ++i) {
strings[i] = std::to_string(std::rand() % 10);
@@ -332,11 +353,16 @@ TEST_F(StringIndexMarisaTest, Codec) {
index->Build(nb, strings.data());
std::vector<std::string> invalid_strings = {std::to_string(nb)};
auto copy_index = milvus::index::CreateStringIndexMarisa();
auto copy_index = milvus::index::CreateStringIndexMarisa(file_manager_ctx);
{
auto binary_set = index->Serialize(nullptr);
copy_index->Load(binary_set);
auto create_index_result = index->UploadV3({});
auto index_files = create_index_result->GetIndexFiles();
Config load_config;
load_config["index_files"] = index_files;
load_config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
copy_index->LoadV3(load_config);
}
{
@@ -398,8 +424,9 @@ TEST_F(StringIndexMarisaTest, Codec) {
}
TEST_F(StringIndexMarisaTest, BaseIndexCodec) {
auto file_manager_ctx = CreateStringTestFileManagerContext();
milvus::index::IndexBasePtr index =
milvus::index::CreateStringIndexMarisa();
milvus::index::CreateStringIndexMarisa(file_manager_ctx);
std::vector<std::string> strings(nb);
for (int i = 0; i < nb; ++i) {
strings[i] = std::to_string(std::rand() % 10);
@@ -410,11 +437,16 @@ TEST_F(StringIndexMarisaTest, BaseIndexCodec) {
index->BuildWithRawDataForUT(str_arr.ByteSizeLong(), data.data());
std::vector<std::string> invalid_strings = {std::to_string(nb)};
auto copy_index = milvus::index::CreateStringIndexMarisa();
auto copy_index = milvus::index::CreateStringIndexMarisa(file_manager_ctx);
{
auto binary_set = index->Serialize(nullptr);
copy_index->Load(binary_set);
auto create_index_result = index->UploadV3({});
auto index_files = create_index_result->GetIndexFiles();
Config load_config;
load_config["index_files"] = index_files;
load_config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
copy_index->LoadV3(load_config);
}
{
internal/core/src/storage/FileManager.h
+6 -4
View File
@@ -277,8 +277,9 @@ class FileManagerImpl : public milvus::FileManager {
virtual std::string
GetRemoteIndexObjectPrefix() const {
boost::filesystem::path prefix =
index::kScalarIndexUseV3 ? "files" : rcm_->GetRootPath();
boost::filesystem::path prefix = index::kOverrideRootPathForUT.empty()
? rcm_->GetRootPath()
: index::kOverrideRootPathForUT;
boost::filesystem::path path = std::string(INDEX_ROOT_PATH);
boost::filesystem::path path1 =
std::to_string(index_meta_.build_id) + "/" +
@@ -293,8 +294,9 @@ class FileManagerImpl : public milvus::FileManager {
if (!stats_base_path_.empty()) {
return stats_base_path_;
}
boost::filesystem::path prefix =
index::kScalarIndexUseV3 ? "files" : rcm_->GetRootPath();
boost::filesystem::path prefix = index::kOverrideRootPathForUT.empty()
? rcm_->GetRootPath()
: index::kOverrideRootPathForUT;
boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH);
boost::filesystem::path path1 =
std::to_string(index_meta_.build_id) + "/" +
internal/core/unittest/init_gtest.cpp
+1 -1
View File
@@ -121,6 +121,6 @@ main(int argc, char** argv) {
std::chrono::milliseconds(0),
std::chrono::milliseconds(-1));
milvus::index::kScalarIndexUseV3 = true;
milvus::index::kOverrideRootPathForUT = "files";
return RUN_ALL_TESTS();
}