add ggml_backend_dev_reset() for sleep mode

ggml/include/ggml-backend.h

@@ -190,6 +190,7 @@ extern "C" {
     GGML_API bool                          ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
     GGML_API bool                          ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);
     GGML_API bool                          ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
+    GGML_API bool                          ggml_backend_dev_reset(ggml_backend_dev_t device);
 
     //
     // Backend (reg)

ggml/src/ggml-backend-impl.h

@@ -199,6 +199,10 @@ extern "C" {
         ggml_backend_event_t (*event_new)         (ggml_backend_dev_t dev);
         void                 (*event_free)        (ggml_backend_dev_t dev, ggml_backend_event_t event);
         void                 (*event_synchronize) (ggml_backend_dev_t dev, ggml_backend_event_t event);
+
+        // (optional) release all resources held for this device (eg. GPU memory/context), the device must remain usable afterwards
+        // returns false if unsupported
+        bool (*reset)(ggml_backend_dev_t dev);
     };
 
     struct ggml_backend_device {

ggml/src/ggml-backend.cpp

@@ -639,6 +639,15 @@ bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_te
     return false;
 }
 
+bool ggml_backend_dev_reset(ggml_backend_dev_t device) {
+    GGML_ASSERT(device);
+    if (device->iface.reset != NULL) {
+        return device->iface.reset(device);
+    }
+
+    return false;
+}
+
 // Backend (reg)
 
 const char * ggml_backend_reg_name(ggml_backend_reg_t reg) {

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -5595,6 +5595,16 @@ static void ggml_backend_cuda_device_event_synchronize(ggml_backend_dev_t dev, g
     CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
 }
 
+static bool ggml_backend_cuda_device_reset(ggml_backend_dev_t dev) {
+    ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
+
+    if (cudaSetDevice(dev_ctx->device) != cudaSuccess) {
+        return false;
+    }
+
+    return cudaDeviceReset() == cudaSuccess;
+}
+
 static const ggml_backend_device_i ggml_backend_cuda_device_interface = {
     /* .get_name                = */ ggml_backend_cuda_device_get_name,
     /* .get_description         = */ ggml_backend_cuda_device_get_description,
@@ -5611,6 +5621,7 @@ static const ggml_backend_device_i ggml_backend_cuda_device_interface = {
     /* .event_new               = */ ggml_backend_cuda_device_event_new,
     /* .event_free              = */ ggml_backend_cuda_device_event_free,
     /* .event_synchronize       = */ ggml_backend_cuda_device_event_synchronize,
+    /* .reset                   = */ ggml_backend_cuda_device_reset,
 };
 
 // backend reg

tools/server/server-context.cpp

@@ -937,7 +937,7 @@ private:
 
     int64_t t_last_load_progress_ms = 0;
 
-    void destroy() {
+    void destroy(bool reset_devices = false) {
         spec.reset();
         ctx_dft.reset();
         model_dft.reset();
@@ -949,13 +949,20 @@ private:
 
         mtmd_free(mctx);
         mctx = nullptr;
+
+        // in sleep mode, we need to reset the devices to free up memory
+        if (reset_devices) {
+            for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+                ggml_backend_dev_reset(ggml_backend_dev_get(i));
+            }
+        }
     }
 
     void handle_sleeping_state(bool new_state) {
         GGML_ASSERT(sleeping != new_state);
         if (new_state) {
             SRV_INF("%s", "server is entering sleeping state\n");
-            destroy();
+            destroy(true);
         } else {
             SRV_INF("%s", "server is exiting sleeping state\n");
             if (!load_model(params_base)) {