enhance: bump scalar index version to V3 as default (#48552)

Set CurrentScalarIndexEngineVersion and MaximumScalarIndexEngineVersion
to 3, enabling V3 scalar index format by default. Update corresponding
test expectations in SegmentLoadInfoTest.

This PR also fixes the issue of snapshot not persisting scalar index
version, as verified by go-sdk e2e tests.

This PR also removes the `V2` series method of
`GetRemoteIndexObjectPrefixV2`, which are the same as their V1
counterparts, except for not prepending root path.

issue: #47417

---------

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>

internal/core/src/index/NgramInvertedIndexTest.cpp

@@ -120,8 +120,6 @@ test_ngram_with_data(const boost::container::vector<std::string>& data,
     auto cm = CreateChunkManager(storage_config);
     auto fs = storage::InitArrowFileSystem(storage_config);
 
-    // Initialize ArrowFileSystemSingleton for AppendIndexV2 which
-    // loads the filesystem from the singleton rather than FileManagerContext.
     auto arrow_fs_conf = milvus_storage::ArrowFileSystemConfig();
     arrow_fs_conf.storage_type = "local";
     arrow_fs_conf.root_path = root_path;
@@ -445,8 +443,6 @@ TEST(NgramIndex, TestNonLikeExpressionsWithNgram) {
     auto cm = CreateChunkManager(storage_config);
     auto fs = storage::InitArrowFileSystem(storage_config);
 
-    // Initialize ArrowFileSystemSingleton for AppendIndexV2 which
-    // loads the filesystem from the singleton rather than FileManagerContext.
     auto arrow_fs_conf = milvus_storage::ArrowFileSystemConfig();
     arrow_fs_conf.storage_type = "local";
     arrow_fs_conf.root_path = root_path;

internal/core/src/index/ScalarIndex.cpp

@@ -196,8 +196,8 @@ ScalarIndex<T>::UploadV3(const Config& config) {
     // Return IndexStats with the single packed file (full remote path)
     std::vector<SerializedIndexFileInfo> index_files;
     auto remote_prefix = is_index_file_
-                             ? file_manager_->GetRemoteIndexObjectPrefixV2()
-                             : file_manager_->GetRemoteTextLogPrefixV2();
+                             ? file_manager_->GetRemoteIndexObjectPrefix()
+                             : file_manager_->GetRemoteTextLogPrefix();
     auto remote_path = remote_prefix + "/" + filename;
     index_files.emplace_back(remote_path, file_size);
 

internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp

@@ -2267,6 +2267,9 @@ ChunkedSegmentSealedImpl::LoadTextIndex(
     }
     milvus::storage::FileManagerContext file_ctx(
         field_data_meta, index_meta, remote_chunk_manager, fs);
+    if (!info_proto->base_path().empty()) {
+        file_ctx.set_stats_base_path(info_proto->base_path());
+    }
 
     auto field_id = milvus::FieldId(info_proto->fieldid());
     // const auto& field_meta = schema_->operator[](field_id);

internal/core/src/segcore/SegmentLoadInfoTest.cpp

@@ -1198,7 +1198,7 @@ TEST_F(SegmentLoadInfoTest, GetLoadDiffTextIndexesFromStats) {
     text_stats.set_version(1);
     text_stats.set_buildid(5001);
     text_stats.set_memory_size(1024);
-    text_stats.set_current_scalar_index_version(2);
+    text_stats.set_current_scalar_index_version(3);
     text_stats.add_files("/path/to/text_index_file1");
     text_stats.add_files("/path/to/text_index_file2");
 
@@ -1221,7 +1221,7 @@ TEST_F(SegmentLoadInfoTest, GetLoadDiffTextIndexesFromStats) {
     EXPECT_EQ(load_info->collectionid(), 200);
     EXPECT_EQ(load_info->partitionid(), 300);
     EXPECT_EQ(load_info->index_size(), 1024);
-    EXPECT_EQ(load_info->current_scalar_index_version(), 2);
+    EXPECT_EQ(load_info->current_scalar_index_version(), 3);
 }
 
 TEST_F(SegmentLoadInfoTest, GetLoadDiffTextIndexesToCreate) {
@@ -1417,7 +1417,7 @@ TEST_F(SegmentLoadInfoTest, ConvertTextIndexStatsToLoadTextIndexInfo) {
     text_stats.set_version(3);
     text_stats.set_buildid(5001);
     text_stats.set_memory_size(2048);
-    text_stats.set_current_scalar_index_version(2);
+    text_stats.set_current_scalar_index_version(3);
     text_stats.add_files("/path/to/file1");
     text_stats.add_files("/path/to/file2");
     text_stats.add_files("/path/to/file3");
@@ -1431,7 +1431,7 @@ TEST_F(SegmentLoadInfoTest, ConvertTextIndexStatsToLoadTextIndexInfo) {
     EXPECT_EQ(load_info->version(), 3);
     EXPECT_EQ(load_info->buildid(), 5001);
     EXPECT_EQ(load_info->index_size(), 2048);
-    EXPECT_EQ(load_info->current_scalar_index_version(), 2);
+    EXPECT_EQ(load_info->current_scalar_index_version(), 3);
     EXPECT_EQ(load_info->files_size(), 3);
     EXPECT_EQ(load_info->files(0), "/path/to/file1");
     EXPECT_EQ(load_info->files(1), "/path/to/file2");

internal/core/src/storage/DiskFileManagerImpl.cpp

@@ -102,7 +102,7 @@ DiskFileManagerImpl::GetRemoteIndexPath(const std::string& file_name,
 
 std::string
 DiskFileManagerImpl::GetRemoteIndexPathV2(const std::string& file_name) const {
-    std::string remote_prefix = GetRemoteIndexObjectPrefixV2();
+    std::string remote_prefix = GetRemoteIndexObjectPrefix();
     return remote_prefix + "/" + file_name;
 }
 

internal/core/src/storage/DiskFileManagerTest.cpp

@@ -272,8 +272,6 @@ TEST_F(DiskAnnFileManagerTest, ReadAndWriteWithStream) {
     lcm->Remove(small_index_file_path);
 }
 
-// Ensure that index v3 generated path is the same as v2 path, only with different
-// file name.
 TEST_F(DiskAnnFileManagerTest, V3PackedIndexPathMismatch) {
     FieldDataMeta filed_data_meta = {1, 2, 3, 100};
     IndexMeta index_meta = {3, 100, 1000, 1, "index"};
@@ -288,11 +286,11 @@ TEST_F(DiskAnnFileManagerTest, V3PackedIndexPathMismatch) {
     ASSERT_EQ(files.size(), 1);
 
     storage::MemFileManagerImpl file_manager(context);
-    std::string v2_path = file_manager.GetRemoteIndexObjectPrefixV2() +
-                          "/milvus_packed_stlsort_index.v3";
+    std::string expected_path = file_manager.GetRemoteIndexObjectPrefix() +
+                                "/milvus_packed_stlsort_index.v3";
     std::string v3_path = files[0];
 
-    EXPECT_EQ(v3_path, v2_path);
+    EXPECT_EQ(v3_path, expected_path);
 }
 
 int

internal/core/src/storage/FileManager.h

@@ -26,6 +26,7 @@
 
 #include "common/Consts.h"
 #include "common/type_c.h"
+#include "index/Meta.h"
 #include "filemanager/FileManager.h"
 #include "log/Log.h"
 #include "milvus-storage/filesystem/fs.h"
@@ -191,8 +192,8 @@ class FileManagerImpl : public milvus::FileManager {
     OpenInputStream(const std::string& filename, bool is_index_file) {
         AssertInfo(fs_, "fs_ is nullptr, cannot open input stream");
         auto local_file_name = GetFileName(filename);
-        auto remote_file_path = is_index_file ? GetRemoteIndexObjectPrefixV2()
-                                              : GetRemoteTextLogPrefixV2();
+        auto remote_file_path = is_index_file ? GetRemoteIndexObjectPrefix()
+                                              : GetRemoteTextLogPrefix();
         remote_file_path += "/" + local_file_name;
         auto remote_file = fs_->OpenInputFile(remote_file_path);
         AssertInfo(remote_file.ok(),
@@ -207,8 +208,8 @@ class FileManagerImpl : public milvus::FileManager {
     OpenOutputStream(const std::string& filename, bool is_index_file) {
         AssertInfo(fs_, "fs_ is nullptr, cannot open output stream");
         auto local_file_name = GetFileName(filename);
-        auto remote_file_path = is_index_file ? GetRemoteIndexObjectPrefixV2()
-                                              : GetRemoteTextLogPrefixV2();
+        auto remote_file_path = is_index_file ? GetRemoteIndexObjectPrefix()
+                                              : GetRemoteTextLogPrefix();
         remote_file_path += "/" + local_file_name;
         // Ensure parent directory exists before opening the output stream.
         // Only needed for local filesystems; object stores don't require
@@ -239,9 +240,8 @@ class FileManagerImpl : public milvus::FileManager {
             auto cipher_plugin = PluginLoader::GetInstance().getCipherPlugin();
             if (cipher_plugin) {
                 auto local_file_name = GetFileName(filename);
-                auto remote_path = is_index_file
-                                       ? GetRemoteIndexObjectPrefixV2()
-                                       : GetRemoteTextLogPrefixV2();
+                auto remote_path = is_index_file ? GetRemoteIndexObjectPrefix()
+                                                 : GetRemoteTextLogPrefix();
                 remote_path += "/" + local_file_name;
                 return std::make_unique<IndexEntryEncryptedLocalWriter>(
                     remote_path,
@@ -277,7 +277,8 @@ class FileManagerImpl : public milvus::FileManager {
 
     virtual std::string
     GetRemoteIndexObjectPrefix() const {
-        boost::filesystem::path prefix = rcm_->GetRootPath();
+        boost::filesystem::path prefix =
+            index::kScalarIndexUseV3 ? "files" : rcm_->GetRootPath();
         boost::filesystem::path path = std::string(INDEX_ROOT_PATH);
         boost::filesystem::path path1 =
             std::to_string(index_meta_.build_id) + "/" +
@@ -287,21 +288,13 @@ class FileManagerImpl : public milvus::FileManager {
         return NormalizePath(prefix / path / path1);
     }
 
-    virtual std::string
-    GetRemoteIndexObjectPrefixV2() const {
-        return std::string(INDEX_ROOT_PATH) + "/" +
-               std::to_string(index_meta_.build_id) + "/" +
-               std::to_string(index_meta_.index_version) + "/" +
-               std::to_string(field_meta_.partition_id) + "/" +
-               std::to_string(field_meta_.segment_id);
-    }
-
     virtual std::string
     GetRemoteTextLogPrefix() const {
         if (!stats_base_path_.empty()) {
             return stats_base_path_;
         }
-        boost::filesystem::path prefix = rcm_->GetRootPath();
+        boost::filesystem::path prefix =
+            index::kScalarIndexUseV3 ? "files" : rcm_->GetRootPath();
         boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH);
         boost::filesystem::path path1 =
             std::to_string(index_meta_.build_id) + "/" +
@@ -313,20 +306,6 @@ class FileManagerImpl : public milvus::FileManager {
         return NormalizePath(prefix / path / path1);
     }
 
-    virtual std::string
-    GetRemoteTextLogPrefixV2() const {
-        if (!stats_base_path_.empty()) {
-            return stats_base_path_;
-        }
-        return std::string(TEXT_LOG_ROOT_PATH) + "/" +
-               std::to_string(index_meta_.build_id) + "/" +
-               std::to_string(index_meta_.index_version) + "/" +
-               std::to_string(field_meta_.collection_id) + "/" +
-               std::to_string(field_meta_.partition_id) + "/" +
-               std::to_string(field_meta_.segment_id) + "/" +
-               std::to_string(field_meta_.field_id);
-    }
-
     static std::string
     GetFileName(const std::string& filepath) {
         return boost::filesystem::path(filepath).filename().string();

internal/core/unittest/init_gtest.cpp

@@ -121,6 +121,6 @@ main(int argc, char** argv) {
         std::chrono::milliseconds(0),
         std::chrono::milliseconds(-1));
 
-    milvus::index::kScalarIndexUseV3 = false;
+    milvus::index::kScalarIndexUseV3 = true;
     return RUN_ALL_TESTS();
 }

internal/datacoord/snapshot.go

@@ -286,6 +286,8 @@ type AvroTextIndexStats struct {
 	MemorySize int64 `avro:"memory_size"`
 	// BuildID is the index build task identifier.
 	BuildID int64 `avro:"build_id"`
+	// CurrentScalarIndexVersion is the scalar index version used for this text index.
+	CurrentScalarIndexVersion int32 `avro:"current_scalar_index_version"`
 }
 
 // AvroJsonKeyStats represents datapb.JsonKeyStats in Avro-compatible format.
@@ -1192,12 +1194,13 @@ func convertTextIndexStatsToAvro(stats *datapb.TextIndexStats) *AvroTextIndexSta
 		return nil
 	}
 	return &AvroTextIndexStats{
-		FieldID:    stats.GetFieldID(),
-		Version:    stats.GetVersion(),
-		Files:      stats.GetFiles(),
-		LogSize:    stats.GetLogSize(),
-		MemorySize: stats.GetMemorySize(),
-		BuildID:    stats.GetBuildID(),
+		FieldID:                   stats.GetFieldID(),
+		Version:                   stats.GetVersion(),
+		Files:                     stats.GetFiles(),
+		LogSize:                   stats.GetLogSize(),
+		MemorySize:                stats.GetMemorySize(),
+		BuildID:                   stats.GetBuildID(),
+		CurrentScalarIndexVersion: stats.GetCurrentScalarIndexVersion(),
 	}
 }
 
@@ -1207,12 +1210,13 @@ func convertAvroToTextIndexStats(avroStats *AvroTextIndexStats) *datapb.TextInde
 		return nil
 	}
 	return &datapb.TextIndexStats{
-		FieldID:    avroStats.FieldID,
-		Version:    avroStats.Version,
-		Files:      avroStats.Files,
-		LogSize:    avroStats.LogSize,
-		MemorySize: avroStats.MemorySize,
-		BuildID:    avroStats.BuildID,
+		FieldID:                   avroStats.FieldID,
+		Version:                   avroStats.Version,
+		Files:                     avroStats.Files,
+		LogSize:                   avroStats.LogSize,
+		MemorySize:                avroStats.MemorySize,
+		BuildID:                   avroStats.BuildID,
+		CurrentScalarIndexVersion: avroStats.CurrentScalarIndexVersion,
 	}
 }
 
@@ -1426,7 +1430,8 @@ func getProperAvroSchema() string {
 								{"name": "index_version", "type": "long"},
 								{"name": "num_rows", "type": "long"},
 								{"name": "current_index_version", "type": "int"},
-								{"name": "mem_size", "type": "long"}
+								{"name": "mem_size", "type": "long"},
+								{"name": "current_scalar_index_version", "type": "int", "default": 0}
 							]
 						}
 					}

pkg/common/common.go

@@ -95,10 +95,8 @@ const (
 
 const (
 	MinimalScalarIndexEngineVersion = int32(0)
-	// TODO: scalar index version 3 is still in development, so we use 2 as the current version.
-	// Do not use version 3 until this TODO is resolved.
-	CurrentScalarIndexEngineVersion = int32(2)
-	MaximumScalarIndexEngineVersion = int32(2)
+	CurrentScalarIndexEngineVersion = int32(3)
+	MaximumScalarIndexEngineVersion = int32(3)
 )
 
 // ClampScalarIndexVersion clamps the given scalar index version to MaximumScalarIndexEngineVersion.