enhance: moved more segcore unit test files (#44210)

issue: https://github.com/milvus-io/milvus/issues/43931

---------

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>

internal/core/src/common/SpanTest.cpp

@@ -11,11 +11,30 @@
 
 #include <gtest/gtest.h>
 
+#include "common/Types.h"
+#include "common/Utils.h"
+#include "common/Span.h"
+#include "common/VectorTrait.h"
 #include "segcore/SegmentGrowing.h"
 #include "test_utils/DataGen.h"
 
 const int64_t ROW_COUNT = 100 * 1000;
 
+TEST(Common, Span) {
+    using namespace milvus;
+    using namespace milvus::segcore;
+
+    Span<float> s1(nullptr, nullptr, 100);
+    Span<milvus::FloatVector> s2(nullptr, 10, 16 * sizeof(float));
+    SpanBase b1 = s1;
+    SpanBase b2 = s2;
+    auto r1 = static_cast<Span<float>>(b1);
+    auto r2 = static_cast<Span<milvus::FloatVector>>(b2);
+    ASSERT_EQ(r1.row_count(), 100);
+    ASSERT_EQ(r2.row_count(), 10);
+    ASSERT_EQ(r2.element_sizeof(), 16 * sizeof(float));
+}
+
 TEST(Span, Naive) {
     using namespace milvus;
     using namespace milvus::query;

internal/core/src/common/UtilsTest.cpp

@@ -31,3 +31,14 @@ TEST(Util_Common, GetCommonPrefix) {
     common_prefix = milvus::GetCommonPrefix(str1, str2);
     EXPECT_STREQ(common_prefix.c_str(), "");
 }
+
+TEST(SimilarityCorelation, Naive) {
+    ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::IP));
+    ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::COSINE));
+
+    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::L2));
+    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::HAMMING));
+    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::JACCARD));
+    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::SUBSTRUCTURE));
+    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::SUPERSTRUCTURE));
+}

internal/core/src/exec/expression/ExprCacheTest.cpp

@@ -57,6 +57,10 @@ TEST(ExprResCacheManagerTest, PutGetBasic) {
     ASSERT_EQ(got.result->size(), 128);
     ASSERT_TRUE(got.valid_result);
     ASSERT_EQ(got.valid_result->size(), 128);
+
+    // restore global state
+    mgr.Clear();
+    ExprResCacheManager::SetEnabled(false);
 }
 
 TEST(ExprResCacheManagerTest, LruEvictionByCapacity) {
@@ -81,6 +85,10 @@ TEST(ExprResCacheManagerTest, LruEvictionByCapacity) {
     ASSERT_FALSE(mgr.Get({1, "expr:0"}, out));
     ASSERT_TRUE(mgr.Get({1, "expr:1"}, out));
     ASSERT_TRUE(mgr.Get({1, "expr:2"}, out));
+
+    // restore global state
+    mgr.Clear();
+    ExprResCacheManager::SetEnabled(false);
 }
 
 TEST(ExprResCacheManagerTest, EraseSegment) {
@@ -106,6 +114,10 @@ TEST(ExprResCacheManagerTest, EraseSegment) {
     ASSERT_FALSE(mgr.Get(k1, out));
     ASSERT_FALSE(mgr.Get(k2, out));
     ASSERT_TRUE(mgr.Get(k3, out));
+
+    // restore global state
+    mgr.Clear();
+    ExprResCacheManager::SetEnabled(false);
 }
 
 TEST(ExprResCacheManagerTest, EnableDisable) {
@@ -127,4 +139,8 @@ TEST(ExprResCacheManagerTest, EnableDisable) {
     ExprResCacheManager::SetEnabled(true);
     mgr.Put(k, v);
     ASSERT_TRUE(mgr.Get(k, out));
+
+    // restore global state
+    mgr.Clear();
+    ExprResCacheManager::SetEnabled(false);
 }

internal/core/src/exec/expression/ExprTest.cpp

@@ -4234,12 +4234,15 @@ TEST_P(ExprTest, TestMutiInConvert) {
     auto plan =
         std::make_shared<plan::FilterBitsNode>(DEFAULT_PLANNODE_ID, expr);
     auto final1 = ExecuteQueryExpr(plan, seg.get(), N, MAX_TIMESTAMP);
+    auto prev_optimize_expr_enabled = OPTIMIZE_EXPR_ENABLED.load();
     OPTIMIZE_EXPR_ENABLED.store(false);
     auto final2 = ExecuteQueryExpr(plan, seg.get(), N, MAX_TIMESTAMP);
     EXPECT_EQ(final1.size(), final2.size());
     for (auto i = 0; i < final1.size(); i++) {
         EXPECT_EQ(final1[i], final2[i]);
     }
+    OPTIMIZE_EXPR_ENABLED.store(prev_optimize_expr_enabled,
+                                std::memory_order_release);
 }
 
 TEST(Expr, TestExprPerformance) {

internal/core/unittest/CMakeLists.txt

@@ -33,129 +33,59 @@ file(GLOB_RECURSE SOURCE_TEST_FILES
 set(MILVUS_TEST_FILES
         ${SOURCE_TEST_FILES}
         init_gtest.cpp
-        test_packed_c.cpp
-        test_arrow_fs_c.cpp
-        test_column_groups_c.cpp
-        test_always_true_expr.cpp
-        test_array_bitmap_index.cpp
-        test_array_inverted_index.cpp
-        test_bf.cpp
-        test_bf_sparse.cpp
-        test_binary.cpp
-        test_binlog_index.cpp
-        test_bool_index.cpp
         test_c_api.cpp
-        test_chunk.cpp
-        test_chunk_vector.cpp
-        test_common.cpp
-        test_concurrent_vector.cpp
-        test_c_stream_reduce.cpp
-        test_c_tokenizer.cpp
         test_loading.cpp
-        test_data_codec.cpp
-        test_delete_record.cpp
-        test_disk_file_manager_test.cpp
         test_exec.cpp
-        test_expr.cpp
         test_expr_materialized_view.cpp
-        test_file_writer.cpp
         test_float16.cpp
-        test_function.cpp
-        test_futures.cpp
         test_group_by.cpp
         test_iterative_filter.cpp
-        test_growing.cpp
-        test_growing_index.cpp
-        test_hybrid_index.cpp
-        test_index_c_api.cpp
         test_indexing.cpp
         test_index_wrapper.cpp
-        test_init.cpp
         test_integer_overflow.cpp
-        test_inverted_index.cpp
-        test_local_chunk_manager.cpp
-        test_mmap_chunk_manager.cpp
-        test_monitor.cpp
-        test_offset_ordered_array.cpp
-        test_offset_ordered_map.cpp
-        test_plan_proto.cpp
         test_query.cpp
-        test_range_search_sort.cpp
-        test_regex_query.cpp
-        test_regex_query_util.cpp
-        test_relational.cpp
-        test_retrieve.cpp
-        test_scalar_index.cpp
         test_sealed.cpp
-        test_segcore.cpp
-        test_similarity_corelation.cpp
         test_storage.cpp
         test_string_expr.cpp
-        test_text_match.cpp
-        test_timestamp_index.cpp
         test_tracer.cpp
-        test_chunked_segment.cpp
-        test_chunked_column.cpp
         test_rust_result.cpp
-        test_cached_search_iterator.cpp
-        test_random_sample.cpp
-        test_json_index.cpp
-        test_types.cpp
-        test_growing_storage_v2.cpp
-        test_memory_planner.cpp
         test_storage_v2_index_raw_data.cpp
-        test_chunked_column_group.cpp
-        test_group_chunk_translator.cpp
-        test_chunked_segment_storage_v2.cpp
-        test_expr_cache.cpp
-        test_json_flat_index.cpp
-        test_vector_array.cpp
-        test_ngram_query.cpp
-        test_stlsort_index.cpp
-        test_vector_array_storage_v2.cpp
-        test_rescore.cpp
-        test_vector_array_chunk.cpp
         )
 
-if ( INDEX_ENGINE STREQUAL "cardinal" )
-    set(MILVUS_TEST_FILES
-        ${MILVUS_TEST_FILES}
-        test_kmeans_clustering.cpp)
+if ( NOT (INDEX_ENGINE STREQUAL "cardinal") )
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "KmeansClusteringTest\\.cpp$")
 endif()
 
-if ( BUILD_DISK_ANN STREQUAL "ON" )
-    set(MILVUS_TEST_FILES
-            ${MILVUS_TEST_FILES}
-            #need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/1757
-            #test_minio_chunk_manager.cpp
-            )
-endif()
+# need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/1757.
+# now we always remove this file from MILVUS_TEST_FILES thus it is never compiled.
+# once done, compile this test file only if `BUILD_DISK_ANN STREQUAL "ON"`.
+# if ( BUILD_DISK_ANN STREQUAL "OFF" )
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "MinioChunkManagerTest\\.cpp$")
+# endif()
 
-if (LINUX OR APPLE)
-    set(MILVUS_TEST_FILES
-            ${MILVUS_TEST_FILES}
-            test_scalar_index_creator.cpp
-            test_string_index.cpp
-            test_array.cpp
-            test_array_expr.cpp)
+# bitset has its own test binary
+list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "BitsetTest\\.cpp$")
+
+if (NOT (LINUX OR APPLE))
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "ScalarIndexCreatorTest\\.cpp$")
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "StringIndexTest\\.cpp$")
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "ArrayTest\\.cpp$")
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "ExprArrayTest\\.cpp$")
 endif()
 
 if (DEFINED AZURE_BUILD_DIR)
-    set(MILVUS_TEST_FILES
-                ${MILVUS_TEST_FILES}
-                test_azure_chunk_manager.cpp
-            #need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119
-                #test_remote_chunk_manager.cpp
-                )
     include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include")
+else()
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "AzureChunkManagerTest\\.cpp$")
 endif()
+# need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119
+# once done, move this line to the else branch of `if (DEFINED AZURE_BUILD_DIR)`
+list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "RemoteChunkManagerTest\\.cpp$")
 
 if (ENABLE_GCP_NATIVE)
     add_definitions(-DENABLE_GCP_NATIVE)
-    set(MILVUS_TEST_FILES
-                ${MILVUS_TEST_FILES}
-                test_gcp_native_chunk_manager.cpp
-                )
+else()
+    list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "GcpNativeChunkManagerTest\\.cpp$")
 endif()
 
 if (LINUX)
@@ -166,26 +96,6 @@ if (LINUX)
         add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
         add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
     endif()
-
-    # check if memory leak exists in index builder
-    set(INDEX_BUILDER_TEST_FILES
-            test_index_wrapper.cpp
-            test_scalar_index_creator.cpp
-            test_index_c_api.cpp
-            )
-
-    add_executable(index_builder_test
-            ${INDEX_BUILDER_TEST_FILES}
-            )
-
-    target_link_libraries(index_builder_test
-            gtest
-            milvus_core
-            knowhere
-            milvus-storage
-            milvus-common
-            )
-    install(TARGETS index_builder_test DESTINATION unittest)
 endif()
 
 add_executable(all_tests
@@ -206,19 +116,6 @@ install(TARGETS all_tests DESTINATION unittest)
 add_subdirectory(bench)
 add_subdirectory(test_json_stats)
 
-# if (USE_DYNAMIC_SIMD)
-# add_executable(dynamic_simd_test
-#                test_simd.cpp)
-#
-# target_link_libraries(dynamic_simd_test
-#                 milvus_simd
-#                 milvus_log
-#                 gtest
-#                 ${CONAN_LIBS})
-#
-# install(TARGETS dynamic_simd_test DESTINATION unittest)
-# endif()
-
 # bitset unit test
 include(CheckCXXCompilerFlag)
 include(CheckIncludeFileCXX)
@@ -227,14 +124,14 @@ check_cxx_compiler_flag("-march=armv8-a+sve" COMPILER_SUPPORTS_SVE)
 check_include_file_cxx("arm_sve.h" COMPILER_HAS_ARM_SVE_HEADER)
 
 add_executable(bitset_test
-        test_bitset.cpp
+        ${CMAKE_HOME_DIRECTORY}/src/bitset/BitsetTest.cpp
 )
 
 if (COMPILER_SUPPORTS_SVE AND COMPILER_HAS_ARM_SVE_HEADER)
         message(STATUS "SVE support for the bitset library UT is enabled")
         target_compile_definitions(bitset_test PRIVATE BITSET_ENABLE_SVE_SUPPORT=1)
 
-        set_source_files_properties(test_bitset.cpp PROPERTIES COMPILE_FLAGS "-march=armv8-a+sve")
+        set_source_files_properties(${CMAKE_HOME_DIRECTORY}/src/bitset/BitsetTest.cpp PROPERTIES COMPILE_FLAGS "-march=armv8-a+sve")
 else()
         message(STATUS "SVE support for the bitset library UT is disabled")
 endif()

internal/core/unittest/test_binary.cpp

@@ -1,33 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#include <gtest/gtest.h>
-#include "test_utils/DataGen.h"
-
-using namespace milvus;
-using namespace milvus::segcore;
-
-TEST(Binary, Insert) {
-    int64_t N = 100000;
-    auto schema = std::make_shared<Schema>();
-    auto vec_fid = schema->AddDebugField(
-        "vecbin", DataType::VECTOR_BINARY, 128, knowhere::metric::JACCARD);
-    auto i64_fid = schema->AddDebugField("age", DataType::INT64);
-    schema->set_primary_field_id(i64_fid);
-    auto dataset = DataGen(schema, N, 10);
-    auto segment = CreateGrowingSegment(schema, empty_index_meta);
-    auto offset = segment->PreInsert(N);
-    segment->Insert(offset,
-                    N,
-                    dataset.row_ids_.data(),
-                    dataset.timestamps_.data(),
-                    dataset.raw_);
-}

internal/core/unittest/test_common.cpp

@@ -1,31 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#include <gtest/gtest.h>
-#include <segcore/ConcurrentVector.h>
-#include "common/Types.h"
-#include "common/Span.h"
-#include "common/VectorTrait.h"
-
-TEST(Common, Span) {
-    using namespace milvus;
-    using namespace milvus::segcore;
-
-    Span<float> s1(nullptr, nullptr, 100);
-    Span<milvus::FloatVector> s2(nullptr, 10, 16 * sizeof(float));
-    SpanBase b1 = s1;
-    SpanBase b2 = s2;
-    auto r1 = static_cast<Span<float>>(b1);
-    auto r2 = static_cast<Span<milvus::FloatVector>>(b2);
-    ASSERT_EQ(r1.row_count(), 100);
-    ASSERT_EQ(r2.row_count(), 10);
-    ASSERT_EQ(r2.element_sizeof(), 16 * sizeof(float));
-}

internal/core/unittest/test_segcore.cpp

@@ -1,92 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-#include <gtest/gtest.h>
-#include <random>
-#include <string>
-
-#include "segcore/SegmentGrowingImpl.h"
-#include "test_utils/DataGen.h"
-
-using namespace milvus;
-
-namespace {
-static constexpr int64_t seg_id = 101;
-auto
-generate_data(int N) {
-    std::vector<char> raw_data;
-    std::vector<uint64_t> timestamps;
-    std::vector<int64_t> uids;
-    std::default_random_engine er(42);
-    std::normal_distribution<> distribution(0.0, 1.0);
-    std::default_random_engine ei(42);
-
-    for (int i = 0; i < N; ++i) {
-        uids.push_back(10 * N + i);
-        timestamps.push_back(0);
-        // append vec
-        float vec[16];
-        for (auto& x : vec) {
-            x = distribution(er);
-        }
-        raw_data.insert(raw_data.end(),
-                        (const char*)std::begin(vec),
-                        (const char*)std::end(vec));
-        int age = ei() % 100;
-        raw_data.insert(raw_data.end(),
-                        (const char*)&age,
-                        ((const char*)&age) + sizeof(age));
-    }
-    return std::make_tuple(raw_data, timestamps, uids);
-}
-}  // namespace
-
-TEST(SegmentCoreTest, NormalDistributionTest) {
-    using namespace milvus::segcore;
-    auto schema = std::make_shared<Schema>();
-    schema->AddDebugField(
-        "fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
-    schema->AddDebugField("age", DataType::INT32);
-    int N = 100 * 1000;
-    auto [raw_data, timestamps, uids] = generate_data(N);
-    auto segment = CreateGrowingSegment(schema, empty_index_meta);
-    segment->PreInsert(N);
-}
-
-// Test insert column-based data
-TEST(SegmentCoreTest, MockTest2) {
-    using namespace milvus::segcore;
-
-    // schema
-    auto schema = std::make_shared<Schema>();
-    schema->AddDebugField(
-        "fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
-    auto i64_fid = schema->AddDebugField("age", DataType::INT64);
-    schema->set_primary_field_id(i64_fid);
-
-    int N = 10000;  // number of records
-    auto dataset = DataGen(schema, N);
-    auto segment = CreateGrowingSegment(schema, empty_index_meta);
-    auto reserved_begin = segment->PreInsert(N);
-    segment->Insert(reserved_begin,
-                    N,
-                    dataset.row_ids_.data(),
-                    dataset.timestamps_.data(),
-                    dataset.raw_);
-}
-
-TEST(SegmentCoreTest, SmallIndex) {
-    using namespace milvus::segcore;
-    auto schema = std::make_shared<Schema>();
-    schema->AddDebugField(
-        "fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
-    schema->AddDebugField("age", DataType::INT32);
-}

internal/core/unittest/test_similarity_corelation.cpp

@@ -1,25 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#include <gtest/gtest.h>
-
-#include "common/Utils.h"
-
-TEST(SimilarityCorelation, Naive) {
-    ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::IP));
-    ASSERT_TRUE(milvus::PositivelyRelated(knowhere::metric::COSINE));
-
-    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::L2));
-    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::HAMMING));
-    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::JACCARD));
-    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::SUBSTRUCTURE));
-    ASSERT_FALSE(milvus::PositivelyRelated(knowhere::metric::SUPERSTRUCTURE));
-}