From c32c3819f73dea793711479ce458675be4495df4 Mon Sep 17 00:00:00 2001 From: Farmadupe Date: Thu, 9 Jul 2026 17:27:18 +0100 Subject: [PATCH] use mmap to reduce unbounded memory usage for split-tensor graph parallell loading (#2102) --- src/llama-model-loader.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index bc20193e1..6d6cf5cf7 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1078,6 +1078,8 @@ bool llama_model_loader::load_all_data( std::vector host_ptrs; std::vector events; + std::vector> split_mappings(files.size()); + ggml_backend_t cuda_backend = nullptr; if (!use_mmap && !check_tensors) { // When not using mmaped io use async uploads from pinned memory to GPU memory. @@ -1197,17 +1199,21 @@ bool llama_model_loader::load_all_data( const char * buffer_name = ggml_backend_buffer_name(cur->buffer); const bool is_probably_split_mode_graph = std::strncmp(buffer_name, GGML_CUDA_NAME, strlen(GGML_CUDA_NAME)) == 0; if (is_probably_split_mode_graph) { - auto & read_buf = read_bufs[thread_idx]; - if (read_buf.capacity() > n_size) { - read_buf = std::vector>(); + llama_mmap * mapping; + { + std::lock_guard lock(load_mutex); + auto & m = split_mappings[weight->idx]; + if (!m) { + m.reset(new llama_mmap(files.at(weight->idx).get(), 0, ggml_is_numa())); + } + mapping = m.get(); } - read_buf.resize(n_size); - file->seek(weight->offs, SEEK_SET); - file->read_raw(read_buf.data(), n_size); - ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size); - if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) { + uint8_t * data = (uint8_t *) mapping->addr() + weight->offs; + ggml_backend_tensor_set(cur, data, 0, n_size); + if (check_tensors && !ggml_validate_row_data(cur->type, data, n_size)) { throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur))); } + mapping->dontneed_fragment(weight->offs, weight->offs + n_size); return n_size; } #endif