From 893b45a626fd5153bb059b69256022746f79a30e Mon Sep 17 00:00:00 2001
From: Xuan Son Nguyen <son@huggingface.co>
Date: Fri, 3 Jul 2026 17:25:58 +0200
Subject: [PATCH] add ggml_backend_dev_reset() for sleep mode

---
 ggml/include/ggml-backend.h     |  1 +
 ggml/src/ggml-backend-impl.h    |  4 ++++
 ggml/src/ggml-backend.cpp       |  9 +++++++++
 ggml/src/ggml-cuda/ggml-cuda.cu | 11 +++++++++++
 tools/server/server-context.cpp | 11 +++++++++--
 5 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
index 2924fdbe98..1fba6326a0 100644
--- a/ggml/include/ggml-backend.h
+++ b/ggml/include/ggml-backend.h
@@ -190,6 +190,7 @@ extern "C" {
     GGML_API bool                          ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
     GGML_API bool                          ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);
     GGML_API bool                          ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
+    GGML_API bool                          ggml_backend_dev_reset(ggml_backend_dev_t device);
 
     //
     // Backend (reg)
diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
index 9c56ec30c5..06d0a5fecd 100644
--- a/ggml/src/ggml-backend-impl.h
+++ b/ggml/src/ggml-backend-impl.h
@@ -199,6 +199,10 @@ extern "C" {
         ggml_backend_event_t (*event_new)         (ggml_backend_dev_t dev);
         void                 (*event_free)        (ggml_backend_dev_t dev, ggml_backend_event_t event);
         void                 (*event_synchronize) (ggml_backend_dev_t dev, ggml_backend_event_t event);
+
+        // (optional) release all resources held for this device (eg. GPU memory/context), the device must remain usable afterwards
+        // returns false if unsupported
+        bool (*reset)(ggml_backend_dev_t dev);
     };
 
     struct ggml_backend_device {
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index 87615921c0..bb3c7b264d 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -639,6 +639,15 @@ bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_te
     return false;
 }
 
+bool ggml_backend_dev_reset(ggml_backend_dev_t device) {
+    GGML_ASSERT(device);
+    if (device->iface.reset != NULL) {
+        return device->iface.reset(device);
+    }
+
+    return false;
+}
+
 // Backend (reg)
 
 const char * ggml_backend_reg_name(ggml_backend_reg_t reg) {
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 78d2218e55..6285fa64f5 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -5595,6 +5595,16 @@ static void ggml_backend_cuda_device_event_synchronize(ggml_backend_dev_t dev, g
     CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
 }
 
+static bool ggml_backend_cuda_device_reset(ggml_backend_dev_t dev) {
+    ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
+
+    if (cudaSetDevice(dev_ctx->device) != cudaSuccess) {
+        return false;
+    }
+
+    return cudaDeviceReset() == cudaSuccess;
+}
+
 static const ggml_backend_device_i ggml_backend_cuda_device_interface = {
     /* .get_name                = */ ggml_backend_cuda_device_get_name,
     /* .get_description         = */ ggml_backend_cuda_device_get_description,
@@ -5611,6 +5621,7 @@ static const ggml_backend_device_i ggml_backend_cuda_device_interface = {
     /* .event_new               = */ ggml_backend_cuda_device_event_new,
     /* .event_free              = */ ggml_backend_cuda_device_event_free,
     /* .event_synchronize       = */ ggml_backend_cuda_device_event_synchronize,
+    /* .reset                   = */ ggml_backend_cuda_device_reset,
 };
 
 // backend reg
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index bb3b91ab5e..6838dee1b3 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -937,7 +937,7 @@ private:
 
     int64_t t_last_load_progress_ms = 0;
 
-    void destroy() {
+    void destroy(bool reset_devices = false) {
         spec.reset();
         ctx_dft.reset();
         model_dft.reset();
@@ -949,13 +949,20 @@ private:
 
         mtmd_free(mctx);
         mctx = nullptr;
+
+        // in sleep mode, we need to reset the devices to free up memory
+        if (reset_devices) {
+            for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+                ggml_backend_dev_reset(ggml_backend_dev_get(i));
+            }
+        }
     }
 
     void handle_sleeping_state(bool new_state) {
         GGML_ASSERT(sleeping != new_state);
         if (new_state) {
             SRV_INF("%s", "server is entering sleeping state\n");
-            destroy();
+            destroy(true);
         } else {
             SRV_INF("%s", "server is exiting sleeping state\n");
             if (!load_model(params_base)) {