From a3836a5de080ce4bd42d7d20723fd827a933b740 Mon Sep 17 00:00:00 2001
From: Nexesenex <124105151+Nexesenex@users.noreply.github.com>
Date: Mon, 13 Jul 2026 12:27:04 +0200
Subject: [PATCH] Fix: on Windows, fall back to read-buffer for sm graph
 loading (#2121)

This, to avoid an intermediary loading of the split tensors in RAM.

Commit c32c3819f7 (PR 2102) introduced mmap for split-mode graph tensor
loading to reduce memory usage via MADV_DONTNEED. However, on Windows
dontneed_fragment() is a no-op (no madvise equivalent), so the entire
mmap'd file stays resident in RAM until load_all_data() returns.

This causes excessive RAM usage with larger models than the available RAM on Windows.

This PR fixes it by guarding the mmap path with !defined(_WIN32) and using the original read-buffer approach on Windows, where per-tensor memory is bounded to the largest single tensor size.
---
 src/llama-model-loader.cpp | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index e53981887..dc544c8cb 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -1078,7 +1078,9 @@ bool llama_model_loader::load_all_data(
     std::vector<void*> host_ptrs;
     std::vector<ggml_backend_event_t> events;
 
+#if !defined(_WIN32)
     std::vector<std::unique_ptr<llama_mmap>> split_mappings(files.size());
+#endif
 
     ggml_backend_t cuda_backend = nullptr;
     if (!use_mmap && !check_tensors) {
@@ -1199,6 +1201,7 @@ bool llama_model_loader::load_all_data(
         const char * buffer_name = ggml_backend_buffer_name(cur->buffer);
         const bool   is_probably_split_mode_graph = std::strncmp(buffer_name, GGML_CUDA_NAME, strlen(GGML_CUDA_NAME)) == 0;
         if (is_probably_split_mode_graph) {
+#if !defined(_WIN32)
             llama_mmap * mapping;
             {
                 std::lock_guard<std::mutex> lock(load_mutex);
@@ -1215,6 +1218,20 @@ bool llama_model_loader::load_all_data(
             }
             mapping->dontneed_fragment(weight->offs, weight->offs + n_size);
             return n_size;
+#else
+            auto & read_buf = read_bufs[thread_idx];
+            if (read_buf.capacity() > n_size) {
+                read_buf = std::vector<no_init<uint8_t>>();
+            }
+            read_buf.resize(n_size);
+            file->seek(weight->offs, SEEK_SET);
+            file->read_raw(read_buf.data(), n_size);
+            ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
+            if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {
+                throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
+            }
+            return n_size;
+#endif
         }
 #endif
         // rest. Serialized.