enhance: storage v2 chunked column memory size from meta (#43130)

use meta to get chunked column memory size to avoid getting cells
actually from storage.
related: #39173

---------

Signed-off-by: shaoting-huang <shaoting.huang@zilliz.com>

internal/core/src/mmap/ChunkedColumnGroup.h

@@ -105,6 +105,18 @@ class ChunkedColumnGroup {
         return meta->num_fields_;
     }
 
+    size_t
+    memory_size() const {
+        auto meta =
+            static_cast<milvus::segcore::storagev2translator::GroupCTMeta*>(
+                slot_->meta());
+        size_t memory_size = 0;
+        for (auto& size : meta->chunk_memory_size_) {
+            memory_size += size;
+        }
+        return memory_size;
+    }
+
  protected:
     mutable std::shared_ptr<CacheSlot<GroupChunk>> slot_;
     size_t num_chunks_{0};
@@ -201,13 +213,7 @@ class ProxyChunkColumn : public ChunkedColumnInterface {
 
     size_t
     DataByteSize() const override {
-        size_t total_size = 0;
-        for (int64_t i = 0; i < num_chunks(); ++i) {
-            auto group_chunk = group_->GetGroupChunk(i);
-            auto chunk = group_chunk.get()->GetChunk(field_id_);
-            total_size += chunk->Size();
-        }
-        return total_size;
+        return group_->memory_size();
     }
 
     int64_t

internal/core/src/segcore/ChunkedSegmentSealedImpl.cpp

@@ -301,7 +301,8 @@ ChunkedSegmentSealedImpl::load_column_group_data_internal(
                                                load_info.mmap_dir_path,
                                                merged_in_load_list);
         LOG_INFO(
-            "segment {} loads column group {} with field ids {} with num_rows "
+            "segment {} loads column group {} with field ids {} with "
+            "num_rows "
             "{}",
             this->get_segment_id(),
             column_group_id.get(),
@@ -334,6 +335,10 @@ ChunkedSegmentSealedImpl::load_column_group_data_internal(
             load_field_data_common(
                 field_id, column, num_rows, data_type, info.enable_mmap, true);
         }
+
+        if (column_group_id.get() == DEFAULT_SHORT_COLUMN_GROUP_ID) {
+            stats_.mem_size += chunked_column_group->memory_size();
+        }
     }
 }
 
@@ -1912,7 +1917,10 @@ ChunkedSegmentSealedImpl::load_field_data_common(
     }
 
     if (!enable_mmap) {
-        stats_.mem_size += column->DataByteSize();
+        if (!is_proxy_column || is_proxy_column &&
+            field_id.get() != DEFAULT_SHORT_COLUMN_GROUP_ID) {
+            stats_.mem_size += column->DataByteSize();
+        }
         if (!IsVariableDataType(data_type) || IsStringDataType(data_type)) {
             LoadSkipIndex(field_id, data_type, column);
         }

internal/core/unittest/test_group_chunk_translator.cpp

@@ -28,6 +28,7 @@
 #include "milvus-storage/format/parquet/file_reader.h"
 #include "test_utils/DataGen.h"
 #include "segcore/storagev2translator/GroupChunkTranslator.h"
+#include "mmap/ChunkedColumnGroup.h"
 
 #include <memory>
 #include <string>
@@ -112,41 +113,56 @@ TEST_P(GroupChunkTranslatorTest, TestWithMmap) {
         "failed to close file reader when get row group metadata from file: " +
             paths_[0] + " with error: " + status.ToString());
 
-    GroupChunkTranslator translator(segment_id_,
-                                    field_metas,
-                                    column_group_info,
-                                    paths_,
-                                    use_mmap,
-                                    row_group_meta_list,
-                                    schema_->get_field_ids().size(),
-                                    milvus::proto::common::LoadPriority::LOW);
+    auto translator = std::make_unique<GroupChunkTranslator>(segment_id_,
+                                                             field_metas,
+                                                             column_group_info,
+                                                             paths_,
+                                                             use_mmap,
+                                                             row_group_meta_list,
+                                                             schema_->get_field_ids().size(),
+                                                             milvus::proto::common::LoadPriority::LOW);
 
     // num cells
-    EXPECT_EQ(translator.num_cells(), row_group_meta_list[0].size());
+    EXPECT_EQ(translator->num_cells(), row_group_meta_list[0].size());
 
     // cell id of
-    for (size_t i = 0; i < translator.num_cells(); ++i) {
-        EXPECT_EQ(translator.cell_id_of(i), i);
+    for (size_t i = 0; i < translator->num_cells(); ++i) {
+        EXPECT_EQ(translator->cell_id_of(i), i);
     }
 
     // key
-    EXPECT_EQ(translator.key(), "seg_0_cg_0");
+    EXPECT_EQ(translator->key(), "seg_0_cg_0");
 
     // estimated byte size
-    for (size_t i = 0; i < translator.num_cells(); ++i) {
+    for (size_t i = 0; i < translator->num_cells(); ++i) {
         auto [file_idx, row_group_idx] =
-            translator.get_file_and_row_group_index(i);
+            translator->get_file_and_row_group_index(i);
         auto& row_group_meta = row_group_meta_list[file_idx].Get(row_group_idx);
-        auto usage = translator.estimated_byte_size_of_cell(i);
+        auto usage = translator->estimated_byte_size_of_cell(i);
         EXPECT_EQ(usage.memory_bytes,
                   static_cast<int64_t>(row_group_meta.memory_size()));
     }
 
     // getting cells
     std::vector<cachinglayer::cid_t> cids = {0, 1};
-    auto cells = translator.get_cells(cids);
+    auto cells = translator->get_cells(cids);
     EXPECT_EQ(cells.size(), cids.size());
 
+    // Test DataByteSize from meta
+    auto meta = static_cast<GroupCTMeta*>(translator->meta());
+    size_t expected_total_size = 0;
+    for (const auto& chunk_size : meta->chunk_memory_size_) {
+        expected_total_size += chunk_size;
+    }
+    EXPECT_GT(expected_total_size, 0);
+    auto num_cells = translator->num_cells();
+
+    auto chunked_column_group =
+        std::make_shared<ChunkedColumnGroup>(std::move(translator));
+
+    EXPECT_EQ(meta->chunk_memory_size_.size(), num_cells);
+    EXPECT_EQ(expected_total_size, chunked_column_group->memory_size());
+
     // Verify mmap directory and files if in mmap mode
     if (use_mmap) {
         std::string mmap_dir = std::to_string(segment_id_);