furyhawk/milvus
1
0
Fork 0
mirror of https://github.com/milvus-io/milvus.git synced 2026-07-21 10:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: make all c++ ut to write to non-tmp and fixed path (#47715)

Browse Source 
issue: #44452

Previously, unit tests could not run in parallel because:

1. Tests used hardcoded /tmp paths (e.g., /tmp/test-bitmap-index/,
/tmp/milvus/rtree-index/), causing file conflicts between shards.
2. Multiple tests independently initialized and released global
singletons (ArrowFileSystemSingleton, LocalChunkManagerSingleton),
leading to use-after-release crashes when shards ran concurrently.

This PR fixes that by:

- Add centralized init_gtest.cpp that generates shard-aware random test
paths (TestLocalPath/TestRemotePath/TestMmapPath) and initializes all
singletons once.
- Replace hardcoded /tmp paths across ~40 test files with TestLocalPath.
- Remove redundant per-test singleton init/release that conflict across
shards.

This PR also fixes 2 other issues:

- Fix `ConvertToArrowSchema` to use `field_ids_` (insertion-ordered
vector) instead of `fields_` (`unordered_map`) for deterministic column
ordering.
- Add thread barrier in PreparedGeometryTest to prevent TLS address
reuse.

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
Buqian Zheng
2026-02-11 11:04:41 +08:00
committed by GitHub
parent 067ee3102f
commit ac9ac0229f
40 changed files with 242 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/core/src/clustering/KmeansClusteringTest.cpp
+4 -2
View File
@@ -23,6 +23,7 @@
#include "storage/InsertData.h"
#include "clustering/KmeansClustering.h"
#include "storage/LocalChunkManagerSingleton.h"
#include "test_utils/Constants.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "test_utils/storage_test_utils.h"
#include "index/Meta.h"
@@ -186,7 +187,7 @@ test_run() {
auto index_meta =
gen_index_meta(segment_id, field_id, index_build_id, index_version);
std::string root_path = "/tmp/test-kmeans-clustering/";
std::string root_path = TestLocalPath;
auto storage_config = gen_local_storage_config(root_path);
auto cm = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
@@ -206,7 +207,8 @@ test_run() {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto get_binlog_path = [=](int64_t log_id) {
return fmt::format("{}/{}/{}/{}/{}",
return fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
internal/core/src/common/ChunkTest.cpp
+1 -1
View File
@@ -1220,7 +1220,7 @@ TEST(chunk, test_create_group_chunk_with_mmap) {
std::nullopt)};
// Create group chunk with mmap
std::string mmap_file = "/tmp/test_group_chunk_mmap.bin";
std::string mmap_file = TestLocalPath + "test_group_chunk_mmap.bin";
if (boost::filesystem::exists(mmap_file)) {
boost::filesystem::remove(mmap_file);
}
internal/core/src/common/PreparedGeometryTest.cpp
+17 -4
View File
@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <gtest/gtest.h>
#include <atomic>
#include <cmath>
#include <cstddef>
#include <mutex>
@@ -250,18 +251,30 @@ TEST(ThreadLocalGEOSContextTest, DifferentThreadsGetDifferentContexts) {
std::mutex mutex;
std::set<GEOSContextHandle_t> contexts;
auto worker = [&mutex, &contexts]() {
constexpr int num_threads = 4;
// Use a barrier to ensure all threads are alive simultaneously,
// preventing thread-local storage address reuse.
std::atomic<int> ready_count{0};
auto worker = [&]() {
GEOSContextHandle_t ctx = GetThreadLocalGEOSContext();
// Verify context is valid by using it
Geometry point(ctx, "POINT(5 5)");
EXPECT_TRUE(point.IsValid());
std::lock_guard<std::mutex> lock(mutex);
contexts.insert(ctx);
{
std::lock_guard<std::mutex> lock(mutex);
contexts.insert(ctx);
}
// Signal ready and wait for all threads to reach this point
ready_count.fetch_add(1);
while (ready_count.load() < num_threads) {
std::this_thread::yield();
}
};
constexpr int num_threads = 4;
std::vector<std::thread> threads;
threads.reserve(num_threads);
internal/core/src/common/Schema.cpp
+3 -3
View File
@@ -132,9 +132,9 @@ const FieldMeta FieldMeta::RowIdMeta(
const ArrowSchemaPtr
Schema::ConvertToArrowSchema() const {
arrow::FieldVector arrow_fields;
arrow_fields.reserve(fields_.size());
for (const auto& field : fields_) {
const auto& meta = field.second;
arrow_fields.reserve(field_ids_.size());
for (const auto& field_id : field_ids_) {
const auto& meta = fields_.at(field_id);
int dim = IsVectorDataType(meta.get_data_type()) &&
!IsSparseFloatVectorDataType(meta.get_data_type())
? meta.get_dim()
internal/core/src/common/VectorArrayChunkTest.cpp
+2 -1
View File
@@ -38,6 +38,7 @@
#include "common/VectorArray.h"
#include "common/protobuf_utils.h"
#include "gtest/gtest.h"
#include "test_utils/Constants.h"
#include "knowhere/comp/index_param.h"
#include "pb/schema.pb.h"
@@ -300,7 +301,7 @@ TEST_F(VectorArrayChunkTest, TestWriteWithMmap) {
// Create temp file path
std::string temp_file =
"/tmp/test_vector_array_chunk_" +
TestLocalPath + "test_vector_array_chunk_" +
std::to_string(
std::chrono::steady_clock::now().time_since_epoch().count());
internal/core/src/common/VectorArrayStorageV2Test.cpp
+7 -14
View File
@@ -72,6 +72,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "test_utils/storage_test_utils.h"
@@ -107,11 +108,6 @@ class TestVectorArrayStorageV2 : public testing::Test {
segcore::SegcoreConfig::default_config(),
true);
// Initialize file system
auto conf = milvus_storage::ArrowFileSystemConfig();
conf.storage_type = "local";
conf.root_path = "/tmp/test_vector_array_for_storage_v2";
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
@@ -269,11 +265,9 @@ class TestVectorArrayStorageV2 : public testing::Test {
void
TearDown() override {
// Clean up test data directory
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
fs->DeleteDir("/tmp/test_vector_array_for_storage_v2");
milvus_storage::ArrowFileSystemSingleton::GetInstance().Release();
fs->DeleteDir("test_data");
}
protected:
@@ -318,8 +312,7 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndex) {
segment_id, vector_array_field_id.get(), index_build_id, index_version);
// Create storage config pointing to the test data location
auto storage_config =
gen_local_storage_config("/tmp/test_vector_array_for_storage_v2");
auto storage_config = gen_local_storage_config(TestLocalPath);
auto cm = CreateChunkManager(storage_config);
// Create index using storage v2 config
@@ -426,8 +419,7 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndexWithMmap) {
segment_id, vector_array_field_id.get(), index_build_id, index_version);
// Create storage config pointing to the test data location
auto storage_config =
gen_local_storage_config("/tmp/test_vector_array_for_storage_v2");
auto storage_config = gen_local_storage_config(TestLocalPath);
auto cm = CreateChunkManager(storage_config);
// Create index using storage v2 config
@@ -483,8 +475,9 @@ TEST_F(TestVectorArrayStorageV2, BuildEmbListHNSWIndexWithMmap) {
auto load_conf = generate_load_conf(
knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::MAX_SIM_IP, 0);
load_conf["index_files"] = index_files;
load_conf["mmap_filepath"] = "mmap/test_emb_list_index";
load_conf["emb_list_meta_file_path"] = "mmap/test_index_meta";
load_conf["mmap_filepath"] = TestLocalPath + "mmap/test_emb_list_index";
load_conf["emb_list_meta_file_path"] =
TestLocalPath + "mmap/test_index_meta";
load_conf[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
vec_index->Load(milvus::tracer::TraceContext{}, load_conf);
internal/core/src/exec/expression/JsonContainsByStatsTest.cpp
+2 -8
View File
@@ -33,7 +33,6 @@
#include "gtest/gtest.h"
#include "index/IndexStats.h"
#include "index/json_stats/JsonKeyStats.h"
#include "milvus-storage/filesystem/fs.h"
#include "pb/common.pb.h"
#include "pb/plan.pb.h"
#include "pb/schema.pb.h"
@@ -50,6 +49,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/storage_test_utils.h"
using namespace milvus;
@@ -100,12 +100,6 @@ BuildAndLoadJsonKeyStats(const std::vector<std::string>& json_strings,
auto chunk_manager = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(
milvus_storage::ArrowFileSystemConfig{
.root_path = root_path,
.storage_type = "local",
});
auto log_path = fmt::format("/{}/{}/{}/{}/{}/{}",
root_path,
collection_id,
@@ -181,7 +175,7 @@ TEST(JsonContainsByStatsTest, BasicContainsAnyOnArray) {
const int64_t field_id = json_fid.get();
const int64_t build_id = 5001;
const int64_t version_id = 1;
const std::string root_path = "/tmp/test-json-contains-by-stats";
const std::string root_path = TestLocalPath;
auto stats = BuildAndLoadJsonKeyStats(json_raw_data,
json_fid,
internal/core/src/exec/expression/LikeConjunctExprTest.cpp
+3 -9
View File
@@ -40,7 +40,6 @@
#include "index/IndexStats.h"
#include "index/Meta.h"
#include "index/NgramInvertedIndex.h"
#include "milvus-storage/filesystem/fs.h"
#include "pb/plan.pb.h"
#include "pb/schema.pb.h"
#include "pb/segcore.pb.h"
@@ -100,12 +99,6 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) {
auto cm = CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
// Initialize ArrowFileSystemSingleton for UpdateSealedSegmentIndex
milvus_storage::ArrowFileSystemConfig arrow_conf;
arrow_conf.storage_type = "local";
arrow_conf.root_path = storage_config.root_path;
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(arrow_conf);
// Test data: 8 rows
// title conditions: "Database" AND "Design"
// content conditions: "SQL" AND "query"
@@ -209,7 +202,8 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) {
insert_data.SetTimestamps(0, 100);
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto log_path = fmt::format("{}/{}/{}/{}/{}",
auto log_path = fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -249,7 +243,7 @@ TEST(LikeConjunctExpr, TestMultiFieldMultiLikeWithRetrieve) {
load_index_info.field_id = field_id.get();
load_index_info.field_type = DataType::VARCHAR;
load_index_info.enable_mmap = true;
load_index_info.mmap_dir_path = "/tmp/test-like-conjunct-mmap-dir";
load_index_info.mmap_dir_path = TestLocalPath + "mmap";
load_index_info.index_id = index_id;
load_index_info.index_build_id = field_index_build_id;
load_index_info.index_version = index_version;
internal/core/src/index/BitmapIndexArrayTest.cpp
+3 -2
View File
@@ -50,6 +50,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
using namespace milvus::index;
using namespace milvus::indexbuilder;
@@ -242,7 +243,7 @@ class ArrayBitmapIndexTest : public testing::Test {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto log_path = fmt::format("/{}/{}/{}/{}/{}/{}",
"/tmp/test_array_bitmap",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -318,7 +319,7 @@ class ArrayBitmapIndexTest : public testing::Test {
int64_t partition_id = 2;
int64_t segment_id = 3;
int64_t field_id = 101;
std::string root_path = "/tmp/test-bitmap-index/";
std::string root_path = TestLocalPath;
storage::StorageConfig storage_config;
storage_config.storage_type = "local";
internal/core/src/index/BitmapIndexTest.cpp
+4 -3
View File
@@ -49,6 +49,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
using namespace milvus::index;
using namespace milvus::indexbuilder;
@@ -169,7 +170,7 @@ class BitmapIndexTest : public testing::Test {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto log_path = fmt::format("/{}/{}/{}/{}/{}/{}",
"/tmp/test-bitmap-index/",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -210,7 +211,7 @@ class BitmapIndexTest : public testing::Test {
if (is_mmap_) {
config["enable_mmap"] = "true";
config["mmap_filepath"] = fmt::format("/{}/{}/{}/{}/{}",
"/tmp/test-bitmap-index/",
TestLocalPath,
collection_id,
1,
segment_id,
@@ -251,7 +252,7 @@ class BitmapIndexTest : public testing::Test {
int64_t partition_id = 2;
int64_t segment_id = 3;
int64_t field_id = 101;
std::string root_path = "/tmp/test-bitmap-index/";
std::string root_path = TestLocalPath;
storage::StorageConfig storage_config;
storage_config.storage_type = "local";
internal/core/src/index/HybridScalarIndexTest.cpp
+3 -2
View File
@@ -49,6 +49,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
using namespace milvus::index;
using namespace milvus::indexbuilder;
@@ -169,7 +170,7 @@ class HybridIndexTestV1 : public testing::Test {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto log_path = fmt::format("/{}/{}/{}/{}/{}/{}",
"/tmp/test_hybrid/",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -245,7 +246,7 @@ class HybridIndexTestV1 : public testing::Test {
int64_t partition_id = 2;
int64_t segment_id = 3;
int64_t field_id = 101;
std::string root_path = "/tmp/test-bitmap-index";
std::string root_path = TestLocalPath;
storage::StorageConfig storage_config;
storage_config.storage_type = "local";
internal/core/src/index/InvertedIndexTest.cpp
+7 -4
View File
@@ -59,6 +59,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "test_utils/storage_test_utils.h"
@@ -131,7 +132,7 @@ test_run() {
}
}
std::string root_path = "/tmp/test-inverted-index/";
std::string root_path = TestLocalPath;
auto storage_config = gen_local_storage_config(root_path);
auto cm = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
@@ -186,7 +187,8 @@ test_run() {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto get_binlog_path = [=](int64_t log_id) {
return fmt::format("{}/{}/{}/{}/{}",
return fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -534,7 +536,7 @@ test_string() {
default_value->set_string_data("20");
}
std::string root_path = "/tmp/test-inverted-index/";
std::string root_path = TestLocalPath;
auto storage_config = gen_local_storage_config(root_path);
auto cm = storage::CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
@@ -580,7 +582,8 @@ test_string() {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto get_binlog_path = [=](int64_t log_id) {
return fmt::format("{}/{}/{}/{}/{}",
return fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
internal/core/src/index/JsonFlatIndexTest.cpp
+5 -3
View File
@@ -65,6 +65,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
#include "test_utils/cachinglayer_test_utils.h"
#include "test_utils/storage_test_utils.h"
@@ -110,7 +111,7 @@ class JsonFlatIndexTest : public ::testing::Test {
index_meta_ =
gen_index_meta(segment_id, field_id, index_build_id, index_version);
std::string root_path = "/tmp/test-json-flat-index/";
std::string root_path = TestLocalPath;
auto storage_config = gen_local_storage_config(root_path);
cm_ = storage::CreateChunkManager(storage_config);
fs_ = storage::InitArrowFileSystem(storage_config);
@@ -138,7 +139,8 @@ class JsonFlatIndexTest : public ::testing::Test {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto get_binlog_path = [=](int64_t log_id) {
return fmt::format("{}/{}/{}/{}/{}",
return fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -192,7 +194,7 @@ class JsonFlatIndexTest : public ::testing::Test {
void
TearDown() override {
cm_w_.reset();
boost::filesystem::remove_all("/tmp/test-json-flat-index/");
// Cleanup handled by random test path in init_gtest.cpp
}
storage::FieldDataMeta field_meta_;
internal/core/src/index/NgramInvertedIndexTest.cpp
+9 -6
View File
@@ -74,6 +74,7 @@
#include "test_utils/DataGen.h"
#include "test_utils/GenExprProto.h"
#include "test_utils/cachinglayer_test_utils.h"
#include "test_utils/Constants.h"
#include "test_utils/storage_test_utils.h"
using namespace milvus;
@@ -107,7 +108,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
auto index_meta = gen_index_meta(
segment_id, field_id.get(), index_build_id, index_version);
std::string root_path = "/tmp/test-inverted-index/";
std::string root_path = TestLocalPath;
auto storage_config = gen_local_storage_config(root_path);
auto cm = CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
@@ -136,7 +137,8 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto get_binlog_path = [=](int64_t log_id) {
return fmt::format("{}/{}/{}/{}/{}",
return fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -231,7 +233,7 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
load_index_info.field_id = field_id.get();
load_index_info.field_type = DataType::VARCHAR;
load_index_info.enable_mmap = true;
load_index_info.mmap_dir_path = "/tmp/test-ngram-index-mmap-dir";
load_index_info.mmap_dir_path = TestLocalPath + "mmap";
load_index_info.index_id = index_id;
load_index_info.index_build_id = index_build_id;
load_index_info.index_version = index_version;
@@ -432,7 +434,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
auto index_meta = gen_index_meta(
segment_id, field_id.get(), index_build_id, index_version);
std::string root_path = "/tmp/test-inverted-index/";
std::string root_path = TestLocalPath;
auto storage_config = gen_local_storage_config(root_path);
auto cm = CreateChunkManager(storage_config);
auto fs = storage::InitArrowFileSystem(storage_config);
@@ -461,7 +463,8 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto get_binlog_path = [=](int64_t log_id) {
return fmt::format("{}/{}/{}/{}/{}",
return fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -511,7 +514,7 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
load_index_info.field_id = field_id.get();
load_index_info.field_type = DataType::VARCHAR;
load_index_info.enable_mmap = true;
load_index_info.mmap_dir_path = "/tmp/test-ngram-index-mmap-dir";
load_index_info.mmap_dir_path = TestLocalPath + "mmap";
load_index_info.index_id = index_id;
load_index_info.index_build_id = index_build_id;
load_index_info.index_version = index_version;
internal/core/src/index/RTreeIndex.cpp
+6 -2
View File
@@ -44,7 +44,11 @@
namespace milvus::index {
constexpr const char* TMP_RTREE_INDEX_PREFIX = "/tmp/milvus/rtree-index/";
static std::string
GetRTreeTempPrefix() {
auto& lcm = milvus::storage::LocalChunkManagerSingleton::GetInstance();
return lcm.GetChunkManager()->GetRootPath() + "/rtree-index/";
}
// helper to check suffix
static inline bool
@@ -62,7 +66,7 @@ RTreeIndex<T>::InitForBuildIndex(bool is_growing) {
path_ = "";
} else {
auto prefix = disk_file_manager_->GetIndexIdentifier();
path_ = std::string(TMP_RTREE_INDEX_PREFIX) + prefix;
path_ = GetRTreeTempPrefix() + prefix;
boost::filesystem::create_directories(path_);
index_file_path = path_ + "/index_file"; // base path (no ext)
if (boost::filesystem::exists(index_file_path + ".bgi")) {
internal/core/src/index/RTreeIndexTest.cpp
+1 -1
View File
@@ -180,7 +180,7 @@ class RTreeIndexTest : public ::testing::Test {
}
}
}
boost::filesystem::remove_all("/tmp/milvus/rtree-index/");
// TmpPath cleanup handles the test directory
} catch (const std::exception& e) {
// Log error but don't fail the test
std::cout << "Warning: Failed to clean up test files: " << e.what()
internal/core/src/index/RTreeIndexWrapperTest.cpp
+2 -1
View File
@@ -20,6 +20,7 @@
#include <vector>
#include "RTreeIndexWrapper.h"
#include "test_utils/Constants.h"
#include "common/Geometry.h"
#include "geos_c.h"
#include "gtest/gtest.h"
@@ -30,7 +31,7 @@ class RTreeIndexWrapperTest : public ::testing::Test {
void
SetUp() override {
// Create test directory
test_dir_ = "/tmp/rtree_test";
test_dir_ = TestLocalPath + "rtree_test";
std::filesystem::create_directories(test_dir_);
// Initialize GEOS
internal/core/src/index/ScalarIndexTest.cpp
+2 -1
View File
@@ -46,6 +46,7 @@
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/AssertUtils.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "test_utils/storage_test_utils.h"
@@ -79,7 +80,7 @@ auto
GetTempFileManagerCtx(CDataType data_type) {
milvus::storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = "/tmp/local/";
storage_config.root_path = TestLocalPath;
auto chunk_manager = milvus::storage::CreateChunkManager(storage_config);
auto ctx = milvus::storage::FileManagerContext(chunk_manager);
ctx.fieldDataMeta.field_schema.set_data_type(
internal/core/src/index/StringIndexSortTest.cpp
+21 -18
View File
@@ -16,6 +16,7 @@
#include "index/StringIndexSort.h"
#include "pb/plan.pb.h"
#include "pb/schema.pb.h"
#include "test_utils/Constants.h"
#include "test_utils/indexbuilder_test_utils.h"
constexpr int64_t nb = 100;
@@ -45,7 +46,7 @@ TEST_F(StringIndexSortTest, ConstructorMemory) {
TEST_F(StringIndexSortTest, ConstructorMmap) {
Config config;
config["mmap_file_path"] = "/tmp/milvus_test";
config["mmap_file_path"] = TestLocalPath + "milvus_test";
auto index = milvus::index::CreateStringIndexSort({});
ASSERT_NE(index, nullptr);
}
@@ -59,7 +60,7 @@ TEST_F(StringIndexSortTest, BuildMemory) {
TEST_F(StringIndexSortTest, BuildMmap) {
Config config;
config["mmap_file_path"] = "/tmp/milvus_test";
config["mmap_file_path"] = TestLocalPath + "milvus_test";
auto index = milvus::index::CreateStringIndexSort({});
index->Build(strs.size(), strs.data());
ASSERT_EQ(index->Count(), nb);
@@ -87,7 +88,7 @@ TEST_F(StringIndexSortTest, InMemory) {
TEST_F(StringIndexSortTest, InMmap) {
Config config;
config["mmap_file_path"] = "/tmp/milvus_test";
config["mmap_file_path"] = TestLocalPath + "milvus_test";
auto index = milvus::index::CreateStringIndexSort({});
index->Build(nb, strs.data());
@@ -184,7 +185,7 @@ TEST_F(StringIndexSortTest, PrefixMatchMemory) {
TEST_F(StringIndexSortTest, PrefixMatchMmap) {
Config config;
config["mmap_file_path"] = "/tmp/milvus_test";
config["mmap_file_path"] = TestLocalPath + "milvus_test";
auto index = milvus::index::CreateStringIndexSort({});
std::vector<std::string> test_strs = {
@@ -213,7 +214,7 @@ TEST_F(StringIndexSortTest, ReverseLookupMemory) {
TEST_F(StringIndexSortTest, ReverseLookupMmap) {
Config config;
config["mmap_file_path"] = "/tmp/milvus_test";
config["mmap_file_path"] = TestLocalPath + "milvus_test";
auto index = milvus::index::CreateStringIndexSort({});
index->Build(strs.size(), strs.data());
@@ -251,7 +252,7 @@ TEST_F(StringIndexSortTest, SerializeDeserializeMemory) {
TEST_F(StringIndexSortTest, SerializeDeserializeMmap) {
Config config;
config["mmap_file_path"] = "/tmp/milvus_test";
config["mmap_file_path"] = TestLocalPath + "milvus_test";
auto index = milvus::index::CreateStringIndexSort({});
index->Build(strs.size(), strs.data());
@@ -296,7 +297,7 @@ TEST_F(StringIndexSortTest, NullHandlingMemory) {
TEST_F(StringIndexSortTest, NullHandlingMmap) {
Config config;
config["mmap_file_path"] = "/tmp/milvus_test";
config["mmap_file_path"] = TestLocalPath + "milvus_test";
auto index = milvus::index::CreateStringIndexSort({});
std::unique_ptr<bool[]> valid(new bool[nb]);
@@ -340,7 +341,8 @@ TEST_F(StringIndexSortTest, MmapLoadAfterSerialize) {
// Step 2: Load with mmap configuration
Config mmap_config;
mmap_config[MMAP_FILE_PATH] = "/tmp/test_string_index_sort_mmap.idx";
mmap_config[MMAP_FILE_PATH] =
TestLocalPath + "test_string_index_sort_mmap.idx";
auto mmap_index = milvus::index::CreateStringIndexSort({});
mmap_index->Load(binary_set, mmap_config);
@@ -386,7 +388,7 @@ TEST_F(StringIndexSortTest, MmapLoadAfterSerialize) {
auto prefix_binary = prefix_index->Serialize(build_config);
Config prefix_mmap_config;
prefix_mmap_config[MMAP_FILE_PATH] = "/tmp/test_prefix_mmap.idx";
prefix_mmap_config[MMAP_FILE_PATH] = TestLocalPath + "test_prefix_mmap.idx";
auto prefix_mmap_index = milvus::index::CreateStringIndexSort({});
prefix_mmap_index->Load(prefix_binary, prefix_mmap_config);
@@ -401,8 +403,8 @@ TEST_F(StringIndexSortTest, MmapLoadAfterSerialize) {
}
// Clean up temp files
std::remove("/tmp/test_string_index_sort_mmap.idx");
std::remove("/tmp/test_prefix_mmap.idx");
std::remove((TestLocalPath + "test_string_index_sort_mmap.idx").c_str());
std::remove((TestLocalPath + "test_prefix_mmap.idx").c_str());
}
TEST_F(StringIndexSortTest, LoadWithoutAssembleMmap) {
@@ -418,7 +420,8 @@ TEST_F(StringIndexSortTest, LoadWithoutAssembleMmap) {
// Load without assemble using mmap
Config mmap_config;
mmap_config[MMAP_FILE_PATH] = "/tmp/test_load_without_assemble.idx";
mmap_config[MMAP_FILE_PATH] =
TestLocalPath + "test_load_without_assemble.idx";
auto mmap_index = milvus::index::CreateStringIndexSort({});
mmap_index->LoadWithoutAssemble(binary_set, mmap_config);
@@ -432,7 +435,7 @@ TEST_F(StringIndexSortTest, LoadWithoutAssembleMmap) {
ASSERT_EQ(range_bitset.count(), 3); // apple, cat, dog
// Clean up
std::remove("/tmp/test_load_without_assemble.idx");
std::remove((TestLocalPath + "test_load_without_assemble.idx").c_str());
}
} // namespace index
} // namespace milvus
@@ -484,7 +487,7 @@ TEST(StringIndexSortStandaloneTest, StringIndexSortBuildAndSearch) {
// Test Mmap mode
{
milvus::Config config;
config["mmap_file_path"] = "/tmp/milvus_scalar_test";
config["mmap_file_path"] = TestLocalPath + "milvus_scalar_test";
auto index = milvus::index::CreateStringIndexSort({});
index->Build(n, test_data.data());
@@ -546,7 +549,7 @@ TEST(StringIndexSortStandaloneTest, StringIndexSortWithNulls) {
// Mmap mode with nulls
{
milvus::Config config;
config["mmap_file_path"] = "/tmp/milvus_scalar_test";
config["mmap_file_path"] = TestLocalPath + "milvus_scalar_test";
auto index = milvus::index::CreateStringIndexSort({});
index->Build(n, test_data.data(), valid_data.get());
@@ -597,7 +600,7 @@ TEST(StringIndexSortStandaloneTest, StringIndexSortSerialization) {
// Test Mmap mode serialization
{
milvus::Config config;
config["mmap_file_path"] = "/tmp/milvus_scalar_test";
config["mmap_file_path"] = TestLocalPath + "milvus_scalar_test";
auto index = milvus::index::CreateStringIndexSort({});
index->Build(n, test_data.data());
@@ -809,7 +812,7 @@ TEST(StringIndexSortPatternMatchTest, PatternMatchMmap) {
milvus::Config mmap_config;
mmap_config[milvus::index::MMAP_FILE_PATH] =
"/tmp/test_pattern_match_mmap.idx";
TestLocalPath + "test_pattern_match_mmap.idx";
auto mmap_index = milvus::index::CreateStringIndexSort({});
mmap_index->Load(binary_set, mmap_config);
@@ -829,7 +832,7 @@ TEST(StringIndexSortPatternMatchTest, PatternMatchMmap) {
ASSERT_TRUE(bitset[4]);
}
std::remove("/tmp/test_pattern_match_mmap.idx");
std::remove((TestLocalPath + "test_pattern_match_mmap.idx").c_str());
}
TEST(StringIndexSortPatternMatchTest, PatternMatchComplexEscape) {
internal/core/src/index/UtilsTest.cpp
+2 -1
View File
@@ -33,6 +33,7 @@
#include "filemanager/InputStream.h"
#include "gtest/gtest.h"
#include "index/Utils.h"
#include "test_utils/Constants.h"
using namespace milvus;
using namespace milvus::index;
@@ -65,7 +66,7 @@ struct TmpFileWrapperIndexUtilsTest {
TEST(UtilIndex, ReadFromFD) {
auto uuid = boost::uuids::random_generator()();
auto uuid_string = boost::uuids::to_string(uuid);
auto file = std::string("/tmp/") + uuid_string;
auto file = TestLocalPath + uuid_string;
auto tmp_file = TmpFileWrapperIndexUtilsTest(file);
ASSERT_NE(tmp_file.fd, -1);
internal/core/src/segcore/ArrowFsCTest.cpp
+6 -1
View File
@@ -20,11 +20,16 @@
#include "common/common_type_c.h"
#include "gtest/gtest.h"
#include "segcore/arrow_fs_c.h"
#include "test_utils/Constants.h"
TEST(ArrowFileSystemSingleton, LocalArrowFileSystemSingleton) {
const char* path = "/tmp";
const char* path = TestLocalPath.c_str();
CStatus status = InitLocalArrowFileSystemSingleton(path);
EXPECT_EQ(status.error_code, 0);
CleanArrowFileSystemSingleton();
// Reinitialize the singleton so subsequent tests can use it
status = InitLocalArrowFileSystemSingleton(path);
EXPECT_EQ(status.error_code, 0);
}
internal/core/src/segcore/ChunkedSegmentSealedStorageV2Test.cpp
+2 -6
View File
@@ -84,11 +84,7 @@ class TestChunkSegmentStorageV2 : public testing::TestWithParam<bool> {
segcore::SegcoreConfig::default_config(),
true);
// Initialize file system
auto conf = milvus_storage::ArrowFileSystemConfig();
conf.storage_type = "local";
conf.root_path = "/tmp/test_data";
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
// Use globally initialized ArrowFileSystem
auto fs = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
@@ -106,7 +102,7 @@ class TestChunkSegmentStorageV2 : public testing::TestWithParam<bool> {
}
std::vector<std::vector<int>> column_groups = {
{0, 4, 3}, {2}, {1}}; // narrow columns and wide columns
{0, 1, 4}, {2}, {3}}; // narrow columns and wide columns
auto writer_memory = 16 * 1024 * 1024;
auto storage_config = milvus_storage::StorageConfig();
internal/core/src/segcore/IndexCTest.cpp
+2 -2
View File
@@ -366,7 +366,7 @@ TEST(CreateIndexTest, StorageV2) {
auto* storage_config = build_index_info->mutable_storage_config();
storage_config->set_storage_type("remote");
storage_config->set_root_path("/tmp/test_storage");
storage_config->set_root_path(TestLocalPath);
storage_config->set_address("localhost:9000");
storage_config->set_bucket_name("test_bucket");
storage_config->set_access_keyid("test_access_key");
@@ -432,7 +432,7 @@ TEST(VectorMemIndexTest, StorageV2) {
auto* storage_config = build_index_info->mutable_storage_config();
storage_config->set_storage_type("local");
storage_config->set_root_path("/tmp");
storage_config->set_root_path(TestLocalPath);
std::string serialized_info;
build_index_info->SerializeToString(&serialized_info);
internal/core/src/segcore/PackedReaderWriterTest.cpp
+5 -2
View File
@@ -36,6 +36,7 @@
#include "segcore/column_groups_c.h"
#include "segcore/packed_reader_c.h"
#include "segcore/packed_writer_c.h"
#include "test_utils/Constants.h"
TEST(CPackedTest, PackedWriterAndReader) {
std::vector<int64_t> test_data(5);
@@ -64,8 +65,10 @@ TEST(CPackedTest, PackedWriterAndReader) {
ASSERT_TRUE(arrow::ExportSchema(*origin_schema, &c_origin_schema).ok());
const int64_t buffer_size = 10 * 1024 * 1024;
char* path = const_cast<char*>("/tmp");
char* paths[] = {const_cast<char*>("/tmp/0")};
std::string root_path = TestLocalPath;
std::string file_path = TestLocalPath + "0";
char* path = const_cast<char*>(root_path.c_str());
char* paths[] = {const_cast<char*>(file_path.c_str())};
int64_t part_upload_size = 0;
CColumnSplits cgs = NewCColumnSplits();
internal/core/src/segcore/SegmentGrowingStorageV2Test.cpp
+2 -5
View File
@@ -55,6 +55,7 @@
#include "segcore/SegmentGrowing.h"
#include "segcore/SegmentGrowingImpl.h"
#include "segcore/memory_planner.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
using namespace milvus;
@@ -64,10 +65,6 @@ namespace pb = milvus::proto;
class TestGrowingStorageV2 : public ::testing::Test {
void
SetUp() override {
auto conf = milvus_storage::ArrowFileSystemConfig();
conf.storage_type = "local";
conf.root_path = path_;
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
SetUpCommonData();
@@ -144,7 +141,7 @@ class TestGrowingStorageV2 : public ::testing::Test {
std::shared_ptr<arrow::Schema> schema_;
std::shared_ptr<arrow::RecordBatch> record_batch_;
std::shared_ptr<arrow::Table> table_;
std::string path_ = "/tmp";
std::string path_ = TestLocalPath;
std::vector<int64_t> ts_values;
std::vector<int64_t> pk_values;
internal/core/src/segcore/storagev1translator/DefaultValueChunkTranslatorTest.cpp
+3 -2
View File
@@ -36,6 +36,7 @@
#include "pb/schema.pb.h"
#include "segcore/storagev1translator/ChunkTranslator.h"
#include "segcore/storagev1translator/DefaultValueChunkTranslator.h"
#include "test_utils/Constants.h"
using namespace milvus;
using namespace milvus::segcore::storagev1translator;
@@ -45,7 +46,7 @@ class DefaultValueChunkTranslatorTest : public ::testing::TestWithParam<bool> {
void
SetUp() override {
// Create a unique temp directory for mmap tests
temp_dir_ = std::filesystem::temp_directory_path() /
temp_dir_ = std::filesystem::path(TestLocalPath) /
("milvus_param_test_" + std::to_string(segment_id_) + "_" +
std::to_string(reinterpret_cast<uintptr_t>(this)));
std::filesystem::create_directories(temp_dir_);
@@ -600,7 +601,7 @@ class DefaultValueChunkTranslatorMmapTest : public ::testing::Test {
SetUp() override {
// Create a unique temp directory for each test
temp_dir_ =
std::filesystem::temp_directory_path() /
std::filesystem::path(TestLocalPath) /
("milvus_test_" + std::to_string(::testing::UnitTest::GetInstance()
->current_test_info()
->line()));
internal/core/src/segcore/storagev2translator/GroupChunkTranslatorTest.cpp
+4 -7
View File
@@ -49,6 +49,7 @@
#include "segcore/Utils.h"
#include "segcore/storagev2translator/GroupCTMeta.h"
#include "segcore/storagev2translator/GroupChunkTranslator.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
using namespace milvus;
@@ -58,10 +59,6 @@ using namespace milvus::segcore::storagev2translator;
class GroupChunkTranslatorTest : public ::testing::TestWithParam<bool> {
void
SetUp() override {
auto conf = milvus_storage::ArrowFileSystemConfig();
conf.storage_type = "local";
conf.root_path = path_;
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
schema_ = CreateTestSchema();
@@ -110,7 +107,7 @@ class GroupChunkTranslatorTest : public ::testing::TestWithParam<bool> {
SchemaPtr schema_;
milvus_storage::ArrowFileSystemPtr fs_;
std::shared_ptr<arrow::Schema> arrow_schema_;
std::string path_ = "/tmp";
std::string path_ = TestLocalPath;
std::vector<std::string> paths_;
int64_t segment_id_ = 0;
@@ -118,7 +115,7 @@ class GroupChunkTranslatorTest : public ::testing::TestWithParam<bool> {
TEST_P(GroupChunkTranslatorTest, TestWithMmap) {
auto temp_dir =
std::filesystem::temp_directory_path() / "gctt_test_with_mmap";
std::filesystem::path(TestLocalPath) / "gctt_test_with_mmap";
std::filesystem::create_directory(temp_dir);
auto use_mmap = GetParam();
@@ -286,7 +283,7 @@ TEST_P(GroupChunkTranslatorTest, TestMultipleFiles) {
}
auto temp_dir =
std::filesystem::temp_directory_path() / "gctt_test_multiple_files";
std::filesystem::path(TestLocalPath) / "gctt_test_multiple_files";
std::filesystem::create_directory(temp_dir);
auto column_group_info = FieldDataInfo(0, total_rows, temp_dir.string());
internal/core/src/storage/DiskFileManagerTest.cpp
+18 -13
View File
@@ -68,6 +68,7 @@
#include "storage/ThreadPool.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
#include "test_utils/storage_test_utils.h"
@@ -106,7 +107,8 @@ class DiskAnnFileManagerTest : public testing::Test {
TEST_F(DiskAnnFileManagerTest, AddFilePositiveParallel) {
auto lcm = LocalChunkManagerSingleton::GetInstance().GetChunkManager();
std::string indexFilePath = "/tmp/diskann/index_files/1000/index";
std::string indexFilePath =
TestLocalPath + "diskann/index_files/1000/index";
auto exist = lcm->Exist(indexFilePath);
EXPECT_EQ(exist, false);
uint64_t index_size = 50 << 20;
@@ -162,7 +164,7 @@ TEST_F(DiskAnnFileManagerTest, AddFilePositiveParallel) {
TEST_F(DiskAnnFileManagerTest, ReadAndWriteWithStream) {
auto conf = milvus_storage::ArrowFileSystemConfig();
conf.storage_type = "local";
conf.root_path = "/tmp/diskann";
conf.root_path = TestLocalPath + "diskann";
auto result = milvus_storage::CreateArrowFileSystem(conf);
EXPECT_TRUE(result.ok());
@@ -170,13 +172,13 @@ TEST_F(DiskAnnFileManagerTest, ReadAndWriteWithStream) {
auto lcm = LocalChunkManagerSingleton::GetInstance().GetChunkManager();
std::string small_index_file_path =
"/tmp/diskann/index_files/1000/1/2/3/small_index_file";
TestLocalPath + "diskann/index_files/1000/1/2/3/small_index_file";
std::string large_index_file_path =
"/tmp/diskann/index_files/1000/1/2/3/large_index_file";
TestLocalPath + "diskann/index_files/1000/1/2/3/large_index_file";
auto exist = lcm->Exist(large_index_file_path);
std::string index_file_path =
"/tmp/diskann/index_files/1000/1/2/3/index_file";
TestLocalPath + "diskann/index_files/1000/1/2/3/index_file";
boost::filesystem::path localPath(index_file_path);
auto local_file_name = localPath.filename().string();
@@ -245,7 +247,7 @@ TEST_F(DiskAnnFileManagerTest, ReadAndWriteWithStream) {
EXPECT_EQ(read_small_index_size, small_index_size);
EXPECT_EQ(is->Tell(), read_offset);
std::string small_index_file_path_read =
"/tmp/diskann/index_files/1000/1/2/3/small_index_file_read";
TestLocalPath + "diskann/index_files/1000/1/2/3/small_index_file_read";
lcm->CreateFile(small_index_file_path_read);
int fd_read = open(small_index_file_path_read.c_str(), O_WRONLY);
ASSERT_NE(fd_read, -1);
@@ -356,7 +358,7 @@ namespace {
const int64_t kOptFieldId = 123456;
const std::string kOptFieldName = "opt_field_name";
const int64_t kOptFieldDataRange = 1000;
const std::string kOptFieldPath = "/tmp/diskann/opt_field/";
// kOptFieldPath computed inline to avoid static initialization order issue
const size_t kEntityCnt = 1000 * 10;
const FieldDataMeta kOptVecFieldDataMeta = {1, 2, 3, 100};
using OffsetT = uint32_t;
@@ -430,7 +432,8 @@ PrepareInsertData(const int64_t opt_field_data_range) -> std::string {
auto chunk_manager =
storage::CreateChunkManager(get_default_local_storage_config());
std::string path = kOptFieldPath + std::to_string(kOptFieldId);
std::string path =
TestLocalPath + "diskann/opt_field/" + std::to_string(kOptFieldId);
boost::filesystem::remove_all(path);
chunk_manager->Write(path, serialized_data.data(), serialized_data.size());
return path;
@@ -669,7 +672,7 @@ TEST_F(DiskAnnFileManagerTest, CacheRawDataToDiskNullableVector) {
auto serialized_data =
insert_data.Serialize(storage::StorageType::Remote);
std::string insert_file_path = "/tmp/diskann/nullable_" +
std::string insert_file_path = TestLocalPath + "diskann/nullable_" +
vec_type.type_name + "_" +
std::to_string(null_percent);
boost::filesystem::remove_all(insert_file_path);
@@ -897,7 +900,7 @@ TEST_F(DiskAnnFileManagerTest, CacheRawDataToDiskValidDataFile) {
auto serialized_data = insert_data.Serialize(storage::StorageType::Remote);
std::string insert_file_path = "/tmp/diskann/valid_data_test";
std::string insert_file_path = TestLocalPath + "diskann/valid_data_test";
boost::filesystem::remove_all(insert_file_path);
cm_->Write(
insert_file_path, serialized_data.data(), serialized_data.size());
@@ -913,7 +916,8 @@ TEST_F(DiskAnnFileManagerTest, CacheRawDataToDiskValidDataFile) {
auto file_manager = std::make_shared<DiskFileManagerImpl>(
storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_));
std::string valid_data_path = "/tmp/diskann/valid_data_test_output";
std::string valid_data_path =
TestLocalPath + "diskann/valid_data_test_output";
boost::filesystem::remove_all(valid_data_path);
milvus::Config config;
@@ -983,7 +987,7 @@ TEST_F(DiskAnnFileManagerTest, CacheRawDataToDiskNoValidDataForNonNullable) {
auto serialized_data = insert_data.Serialize(storage::StorageType::Remote);
std::string insert_file_path = "/tmp/diskann/non_nullable_test";
std::string insert_file_path = TestLocalPath + "diskann/non_nullable_test";
boost::filesystem::remove_all(insert_file_path);
cm_->Write(
insert_file_path, serialized_data.data(), serialized_data.size());
@@ -999,7 +1003,8 @@ TEST_F(DiskAnnFileManagerTest, CacheRawDataToDiskNoValidDataForNonNullable) {
auto file_manager = std::make_shared<DiskFileManagerImpl>(
storage::FileManagerContext(field_data_meta, index_meta, cm_, fs_));
std::string valid_data_path = "/tmp/diskann/non_nullable_valid_data";
std::string valid_data_path =
TestLocalPath + "diskann/non_nullable_valid_data";
boost::filesystem::remove_all(valid_data_path);
milvus::Config config;
internal/core/src/storage/FileWriterTest.cpp
+5 -17
View File
@@ -27,6 +27,7 @@
#include "common/EasyAssert.h"
#include "gtest/gtest.h"
#include "storage/FileWriter.h"
#include "test_utils/Constants.h"
using namespace milvus;
using namespace milvus::storage;
@@ -35,7 +36,7 @@ class FileWriterTest : public testing::Test {
protected:
void
SetUp() override {
test_dir_ = std::filesystem::temp_directory_path() / "file_writer_test";
test_dir_ = std::filesystem::path(TestLocalPath) / "file_writer_test";
std::filesystem::create_directories(test_dir_);
}
@@ -959,17 +960,12 @@ TEST_F(FileWriterTest, ConcurrentAccessToFileWriterConfig) {
// Verify that the configuration is still valid
FileWriter::SetMode(FileWriter::WriteMode::BUFFERED);
std::string filename =
(std::filesystem::temp_directory_path() / "concurrent_config_test.txt")
.string();
std::string filename = (test_dir_ / "concurrent_config_test.txt").string();
FileWriter writer(filename);
std::string test_data = "Concurrent config test";
writer.Write(test_data.data(), test_data.size());
writer.Finish();
// Clean up
std::filesystem::remove(filename);
}
// Test that changing FileWriterConfig during FileWriter operations doesn't affect existing instances
TEST_F(FileWriterTest, ConfigChangeDuringFileWriterOperations) {
@@ -977,12 +973,8 @@ TEST_F(FileWriterTest, ConfigChangeDuringFileWriterOperations) {
FileWriter::SetMode(FileWriter::WriteMode::BUFFERED);
FileWriteWorkerPool::GetInstance().Configure(2);
std::string filename1 =
(std::filesystem::temp_directory_path() / "config_change_test1.txt")
.string();
std::string filename2 =
(std::filesystem::temp_directory_path() / "config_change_test2.txt")
.string();
std::string filename1 = (test_dir_ / "config_change_test1.txt").string();
std::string filename2 = (test_dir_ / "config_change_test2.txt").string();
// Create first FileWriter
FileWriter writer1(filename1);
@@ -1024,8 +1016,4 @@ TEST_F(FileWriterTest, ConfigChangeDuringFileWriterOperations) {
std::string content2((std::istreambuf_iterator<char>(file2)),
std::istreambuf_iterator<char>());
EXPECT_EQ(content2, test_data2);
// Clean up
std::filesystem::remove(filename1);
std::filesystem::remove(filename2);
}
internal/core/unittest/CMakeLists.txt
+3
View File
@@ -109,6 +109,9 @@ if (LINUX)
endif()
endif()
add_compile_definitions(
MILVUS_CPPUT_OUTPUT_DIR="${CMAKE_INSTALL_PREFIX}/cpput_output")
add_executable(all_tests
${MILVUS_TEST_FILES}
)
internal/core/unittest/init_gtest.cpp
+47
View File
@@ -12,23 +12,65 @@
#include <gtest/gtest.h>
#include <stdint.h>
#include <chrono>
#include <cstdlib>
#include <ctime>
#include <filesystem>
#include <memory>
#include <string>
#include "cachinglayer/Manager.h"
#include "common/common_type_c.h"
#include "exec/expression/function/init_c.h"
#include "folly/init/Init.h"
#include "milvus-storage/filesystem/fs.h"
#include "storage/LocalChunkManagerSingleton.h"
#include "storage/MmapManager.h"
#include "storage/RemoteChunkManagerSingleton.h"
#include "test_utils/Constants.h"
#include "test_utils/storage_test_utils.h"
std::string TestLocalPath;
std::string TestRemotePath;
std::string TestMmapPath;
int
main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
folly::Init follyInit(&argc, &argv, false);
// Determine the base directory for test output.
// Priority: MILVUS_TEST_ROOT_DIR env var > compile-time MILVUS_CPPUT_OUTPUT_DIR
std::string base_dir;
if (auto* env = std::getenv("MILVUS_TEST_ROOT_DIR")) {
base_dir = env;
} else {
base_dir = MILVUS_CPPUT_OUTPUT_DIR;
}
// Compute shard-aware test paths to avoid conflicts in parallel test runs
int shard_index = 0;
int total_shards = 1;
if (auto* env = std::getenv("GTEST_SHARD_INDEX")) {
shard_index = std::atoi(env);
}
if (auto* env = std::getenv("GTEST_TOTAL_SHARDS")) {
total_shards = std::atoi(env);
}
std::srand(std::time(nullptr));
int random = std::rand();
// Ensure random % total_shards == shard_index so different shards use different paths
random = random - (random % total_shards) + shard_index;
TestLocalPath =
base_dir + "/test_" + std::to_string(random) + "/local_data/";
TestRemotePath =
base_dir + "/test_" + std::to_string(random) + "/remote_data/";
TestMmapPath = base_dir + "/test_" + std::to_string(random) + "/mmap_data/";
// Ensure test directories exist before any Init calls
std::filesystem::create_directories(TestLocalPath);
std::filesystem::create_directories(TestRemotePath);
std::filesystem::create_directories(TestMmapPath);
// Initialize expression function factory (registers aggregate functions like count, sum, min, max)
InitExecExpressionFunctionFactory();
@@ -38,6 +80,11 @@ main(int argc, char** argv) {
get_default_local_storage_config());
milvus::storage::MmapManager::GetInstance().Init(get_default_mmap_config());
milvus_storage::ArrowFileSystemConfig arrow_conf;
arrow_conf.storage_type = "local";
arrow_conf.root_path = TestLocalPath;
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(arrow_conf);
static const int64_t mb = 1024 * 1024;
milvus::cachinglayer::Manager::ConfigureTieredStorage(
internal/core/unittest/test_indexing.cpp
+2 -1
View File
@@ -612,7 +612,8 @@ TEST_P(IndexTest, Mmap) {
ASSERT_GT(serializedSize, 0);
load_conf = generate_load_conf(index_type, metric_type, 0);
load_conf["index_files"] = index_files;
load_conf["mmap_filepath"] = "mmap/test_index_mmap_" + index_type;
load_conf["mmap_filepath"] =
TestLocalPath + "mmap/test_index_mmap_" + index_type;
load_conf[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
vec_index->Load(milvus::tracer::TraceContext{}, load_conf);
internal/core/unittest/test_json_stats/test_json_key_stats.cpp
+6 -20
View File
@@ -43,7 +43,6 @@
#include "index/json_stats/bson_inverted.h"
#include "index/json_stats/utils.h"
#include "indexbuilder/IndexCreatorBase.h"
#include "milvus-storage/filesystem/fs.h"
#include "pb/common.pb.h"
#include "pb/schema.pb.h"
#include "simdjson/padded_string.h"
@@ -54,6 +53,7 @@
#include "storage/ThreadPools.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
using namespace milvus::index;
using namespace milvus::indexbuilder;
@@ -153,7 +153,7 @@ class JsonKeyStatsTest : public ::testing::TestWithParam<bool> {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto log_path = fmt::format("/{}/{}/{}/{}/{}/{}",
"/tmp/test-jsonkey-stats/",
TestLocalPath,
collection_id,
partition_id,
segment_id,
@@ -184,8 +184,7 @@ class JsonKeyStatsTest : public ::testing::TestWithParam<bool> {
config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
config[milvus::index::ENABLE_MMAP] = true;
config[milvus::index::MMAP_FILE_PATH] =
"/tmp/test-jsonkey-stats/mmap-file";
config[milvus::index::MMAP_FILE_PATH] = TestLocalPath + "mmap-file";
index_ = std::make_shared<JsonKeyStats>(ctx, true);
index_->Load(milvus::tracer::TraceContext{}, config);
}
@@ -201,17 +200,11 @@ class JsonKeyStatsTest : public ::testing::TestWithParam<bool> {
int64_t index_build_id = GenerateRandomInt64(1, 100000);
int64_t index_version = 1;
size_ = 1000; // Use a larger size for better testing
std::string root_path = "/tmp/test-jsonkey-stats/";
storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = root_path;
storage_config.root_path = TestLocalPath;
chunk_manager_ = storage::CreateChunkManager(storage_config);
auto conf = milvus_storage::ArrowFileSystemConfig();
conf.storage_type = "local";
conf.root_path = "/tmp/test-jsonkey-stats/arrow-fs";
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
@@ -225,7 +218,6 @@ class JsonKeyStatsTest : public ::testing::TestWithParam<bool> {
}
virtual ~JsonKeyStatsTest() override {
boost::filesystem::remove_all(chunk_manager_->GetRootPath());
}
public:
@@ -354,24 +346,19 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test {
field_id_ = 101;
index_build_id_ = GenerateRandomInt64(1, 100000);
index_version_ = 10000;
root_path_ = "/tmp/test-jsonkey-stats-upload-load/";
root_path_ = TestLocalPath;
storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = root_path_;
chunk_manager_ = storage::CreateChunkManager(storage_config);
auto conf = milvus_storage::ArrowFileSystemConfig();
conf.storage_type = "local";
conf.root_path = "/tmp/test-jsonkey-stats-upload-load/arrow-fs";
milvus_storage::ArrowFileSystemSingleton::GetInstance().Init(conf);
fs_ = milvus_storage::ArrowFileSystemSingleton::GetInstance()
.GetArrowFileSystem();
}
void
TearDown() override {
boost::filesystem::remove_all(chunk_manager_->GetRootPath());
}
void
@@ -444,8 +431,7 @@ class JsonKeyStatsUploadLoadTest : public ::testing::Test {
Config config;
config["index_files"] = index_files_;
config[milvus::index::ENABLE_MMAP] = true;
config[milvus::index::MMAP_FILE_PATH] =
"/tmp/test-jsonkey-stats-upload-load/mmap-file";
config[milvus::index::MMAP_FILE_PATH] = TestLocalPath + "mmap-file";
config[milvus::LOAD_PRIORITY] =
milvus::proto::common::LoadPriority::HIGH;
load_index_ = std::make_shared<JsonKeyStats>(ctx, true);
internal/core/unittest/test_json_stats/test_traverse_json_for_build_stats.cpp
+3 -2
View File
@@ -22,6 +22,7 @@
#include "storage/FileManager.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
using milvus::index::JsonKey;
using milvus::index::JsonKeyStats;
@@ -100,7 +101,7 @@ TEST(TraverseJsonForBuildStatsTest,
milvus::storage::IndexMeta index_meta{3, 100, 1, 1};
milvus::storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = "/tmp/test-traverse-json-build-stats";
storage_config.root_path = TestLocalPath;
auto cm = milvus::storage::CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
milvus::storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
@@ -141,7 +142,7 @@ TEST(CollectSingleJsonStatsInfoTest, EmptyJsonStringThrows) {
milvus::storage::IndexMeta index_meta{3, 100, 1, 1};
milvus::storage::StorageConfig storage_config;
storage_config.storage_type = "local";
storage_config.root_path = "/tmp/test-collect-single-json-stats-info";
storage_config.root_path = TestLocalPath;
auto cm = milvus::storage::CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
milvus::storage::FileManagerContext ctx(field_meta, index_meta, cm, fs);
internal/core/unittest/test_sealed.cpp
+4 -2
View File
@@ -78,6 +78,7 @@
#include "storage/RemoteChunkManagerSingleton.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
#include "test_utils/cachinglayer_test_utils.h"
#include "test_utils/storage_test_utils.h"
@@ -2404,7 +2405,7 @@ TEST_P(SealedVectorArrayTest, SearchVectorArray) {
auto dataset = DataGen(schema, dataset_size, 42, 0, 1, emb_list_len);
// create field data
std::string root_path = "/tmp/test-vector-array/";
std::string root_path = TestLocalPath;
auto storage_config = gen_local_storage_config(root_path);
auto cm = CreateChunkManager(storage_config);
auto fs = milvus::storage::InitArrowFileSystem(storage_config);
@@ -2437,7 +2438,8 @@ TEST_P(SealedVectorArrayTest, SearchVectorArray) {
auto serialized_bytes = insert_data.Serialize(storage::Remote);
auto get_binlog_path = [=](int64_t log_id) {
return fmt::format("{}/{}/{}/{}/{}",
return fmt::format("{}{}/{}/{}/{}/{}",
TestLocalPath,
collection_id,
partition_id,
segment_id,
internal/core/unittest/test_storage.cpp
+5 -5
View File
@@ -33,12 +33,12 @@
#include "storage/Types.h"
#include "storage/Util.h"
#include "storage/storage_c.h"
#include "test_utils/Constants.h"
using namespace std;
using namespace milvus;
using namespace milvus::storage;
string rootPath = "files";
string bucketName = "a-bucket";
CStorageConfig
@@ -53,7 +53,7 @@ get_azure_storage_config() {
bucketName.c_str(),
accessKey,
accessValue,
rootPath.c_str(),
TestRemotePath.c_str(),
"remote",
"azure",
"",
@@ -88,7 +88,7 @@ TEST_F(StorageTest, InitLocalChunkManagerSingleton) {
TEST_F(StorageTest, GetLocalUsedSize) {
int64_t size = 0;
auto lcm = LocalChunkManagerSingleton::GetInstance().GetChunkManager();
EXPECT_EQ(lcm->GetRootPath(), "/tmp/milvus/local_data/");
EXPECT_EQ(lcm->GetRootPath(), TestLocalPath);
string test_dir =
lcm->GetRootPath() + "tmp" +
// add random number to avoid dir conflict
@@ -116,7 +116,7 @@ TEST_F(StorageTest, InitRemoteChunkManagerSingleton) {
EXPECT_EQ(status.error_code, Success);
auto rcm =
RemoteChunkManagerSingleton::GetInstance().GetRemoteChunkManager();
EXPECT_EQ(rcm->GetRootPath(), "/tmp/milvus/remote_data");
EXPECT_EQ(rcm->GetRootPath(), TestRemotePath);
}
TEST_F(StorageTest, CleanRemoteChunkManagerSingleton) {
@@ -226,7 +226,7 @@ TEST_F(StorageUtilTest, TestInitArrowFileSystem) {
{
StorageConfig local_config;
local_config.storage_type = "local";
local_config.root_path = "/tmp/milvus/local_data";
local_config.root_path = TestLocalPath;
auto fs = InitArrowFileSystem(local_config);
ASSERT_NE(fs, nullptr);
internal/core/unittest/test_storage_v2_index_raw_data.cpp
+3 -2
View File
@@ -49,6 +49,7 @@
#include "storage/FileManager.h"
#include "storage/Types.h"
#include "storage/Util.h"
#include "test_utils/Constants.h"
#include "test_utils/DataGen.h"
#include "test_utils/storage_test_utils.h"
@@ -67,7 +68,7 @@ class StorageV2IndexRawDataTest : public ::testing::Test {
protected:
storage::ChunkManagerPtr cm_;
milvus_storage::ArrowFileSystemPtr fs_;
std::string path_ = "/tmp/test-inverted-index-storage-v2";
std::string path_ = TestLocalPath;
int64_t collection_id = 1;
int64_t partition_id = 2;
int64_t segment_id = 3;
@@ -175,7 +176,7 @@ TEST_F(StorageV2IndexRawDataTest, TestGetRawData) {
storage::FileManagerContext(
field_data_meta, index_meta, cm_, fs_));
auto res = file_manager->CacheRawDataToDisk<float>(config);
ASSERT_EQ(res, "/tmp/milvus/local_data/raw_datas/3/105/raw_data");
ASSERT_EQ(res, TestLocalPath + "raw_datas/3/105/raw_data");
}
{
internal/core/unittest/test_utils/Constants.h
+11 -2
View File
@@ -12,10 +12,19 @@
#pragma once
#include <cstdint>
#include <string>
constexpr int64_t TestChunkSize = 32 * 1024;
constexpr char TestLocalPath[] = "/tmp/milvus/local_data/";
constexpr char TestRemotePath[] = "/tmp/milvus/remote_data";
// Root path for LocalChunkManager and ArrowFileSystem in tests.
// Initialized in init_gtest.cpp with a shard-aware random path to
// avoid conflicts during parallel test execution.
extern std::string TestLocalPath;
// Root path for RemoteChunkManager in tests (simulated as local storage).
// Shares the same random prefix as TestLocalPath.
extern std::string TestRemotePath;
// Root path for MmapManager tests. Separate from TestLocalPath to avoid
// being deleted when tests clean up LocalChunkManager's root path.
extern std::string TestMmapPath;
constexpr int64_t kTestSparseDim = 1000;
constexpr float kTestSparseVectorDensity = 0.003;
internal/core/unittest/test_utils/TmpPath.h
+4 -2
View File
@@ -12,12 +12,14 @@
#include <boost/filesystem.hpp>
#include <string>
#include "test_utils/Constants.h"
namespace milvus::test {
struct TmpPath {
TmpPath() {
temp_path_ = boost::filesystem::temp_directory_path() /
temp_path_ = boost::filesystem::path(TestLocalPath) /
boost::filesystem::unique_path();
boost::filesystem::create_directory(temp_path_);
boost::filesystem::create_directories(temp_path_);
}
~TmpPath() {
boost::filesystem::remove_all(temp_path_);
internal/core/unittest/test_utils/storage_test_utils.h
+3 -2
View File
@@ -66,7 +66,7 @@ inline MmapConfig
get_default_mmap_config() {
MmapConfig mmap_config = {
.cache_read_ahead_policy = "willneed",
.mmap_path = "/tmp/test_mmap_manager/",
.mmap_path = TestMmapPath,
.disk_limit =
uint64_t(2) * uint64_t(1024) * uint64_t(1024) * uint64_t(1024),
.fix_file_size = uint64_t(4) * uint64_t(1024) * uint64_t(1024),
@@ -170,7 +170,8 @@ PrepareSingleFieldInsertBinlog(int64_t collection_id,
for (auto i = 0; i < field_datas.size(); ++i) {
auto& field_data = field_datas[i];
row_count += field_data->get_num_rows();
auto file = "./data/test/" + std::to_string(collection_id) + "/" +
auto file = TestRemotePath + "data/test/" +
std::to_string(collection_id) + "/" +
std::to_string(partition_id) + "/" +
std::to_string(segment_id) + "/" +
std::to_string(field_id) + "/" + std::to_string(i);