on-demand tensor reload (#1989)

* host-swap tensor loop

the host-swap functionality is only triggered when the certain env. variables are declared

* target_include_directories tweak

* hot-swap tensor support

two intrusions:
1.) at the model loading to collect the snapshot
2.) the modification of the `/health` HTTP endpoint to be able to trigger the hot-swap via sending the `llama-server` the HTTP-request.
*both a braced by the specific env. variables

* hot-swap tensor support; graph invalidation

ggml_backend_cuda_invalidate_graphs export

* hot-swap tensor support

graph invalidation implementation;  extended debug output (commented out)

* llama_reload_changed_tensors export

* tensor hot-swap on-demand reload

cpu-only/hybrid/gpu-only with split mode layer/graph full support implementation

* docs

* reuse the gguf parsing from llama.cpp

gguf_init_from_file, gguf_find_tensor, ggml_get_tensor

* remove the manual scheduling for hybrid inference

* update docs

* tensor shape validation

* update docs

* update docs

accidentally wiped the previous changes;  so recovered them

* revert the GGML_CUDA_MAX_DEVICES to 16

* update llama_reload_changed_tensor

update llama_reload_changed_tensor, revert CMakeLists.txt

* update llama_reload_changed_tensor

* GGML_MAX_SRC

GGML_MAX_SRC compile-time definition support

* GGML_MAX_SRC

GGML_MAX_SRC compile-time definition support

* GGML_MAX_SRC

GGML_MAX_SRC compile-time definition support

* llama_reload_changed_tensor

update llama_reload_changed_tensor definition

* refactory

move the tensor-reloading implementation to llama-reload.cpp, llama-reload-info.h;  some bugfixes and code reduction

* revert

added back the missing newline

* update docs

* reload_info constructor

* bugfix: cpu-only

TODO: improve the working environment by compiling for multiple hardware configurations;  possibly make a test pipeline

* cpu-only bugfix

set the fix again after unsuccessful sync with main

* windows os compilation fix

#include <string>

* fix windows os build

error C2039: 'string': is not a member of 'std'

* remove dead file

* implement perplexity in server

* Revert "implement perplexity in server"

examples/perplexity/perplexity.cpp

@@ -20,11 +20,15 @@
 #include <array>
 #include <fstream>
 #include <sstream>
+#include <cstdlib>
 
 #if defined(_MSC_VER)
 #pragma warning(disable: 4244 4267) // possible loss of data
 #endif
 
+// Public C API for hot-swap (defined in src/llama.cpp)
+extern "C" bool llama_reload_changed_tensors(struct llama_context * ctx);
+
 struct results_perplexity {
     std::vector<llama_token> tokens;
     double                   ppl_value;
@@ -2056,21 +2060,62 @@ int main(int argc, char ** argv) {
         fprintf(stderr, "%s\n", gpt_params_get_system_info(params).c_str());
     }
 
-    struct results_perplexity results;
-    if (params.hellaswag) {
-        hellaswag_score(ctx, params);
-    } else if (params.winogrande) {
-        winogrande_score(ctx, params);
-    } else if (params.multiple_choice) {
-        multiple_choice_score(ctx, params);
-    } else if (params.kl_divergence) {
-        kl_divergence(ctx, params);
-    } else {
-        results = perplexity(ctx, params, n_ctx);
+    const char * hotswap_env = std::getenv("LLAMA_HOTSWAP_ENABLED");
+    const char * pre_script  = std::getenv("LLAMA_PERPLEXITY_PRE_RELOAD_SCRIPT");
+
+    if (hotswap_env) {
+        llama_reload_changed_tensors(ctx);
     }
 
-    llama_print_timings(ctx);
-    write_logfile(ctx, params, model, results);
+    while (true) {
+        struct results_perplexity results;
+        if (params.hellaswag) {
+            hellaswag_score(ctx, params);
+        } else if (params.winogrande) {
+            winogrande_score(ctx, params);
+        } else if (params.multiple_choice) {
+            multiple_choice_score(ctx, params);
+        } else if (params.kl_divergence) {
+            kl_divergence(ctx, params);
+        } else {
+            results = perplexity(ctx, params, n_ctx);
+        }
+
+        llama_print_timings(ctx);
+        write_logfile(ctx, params, model, results);
+
+        if (pre_script) {
+            fprintf(stderr, "%s: executing pre-reload script: %s\n", __func__, pre_script);
+#ifdef _WIN32
+            FILE * fp = _popen(pre_script, "r");
+#else
+            FILE * fp = popen(pre_script, "r");
+#endif
+            if (fp) {
+                char buf[256];
+                while (fgets(buf, sizeof(buf), fp)) {
+                    size_t len = strlen(buf);
+                    if (len > 0 && buf[len-1] == '\n') buf[len-1] = '\0';
+                    fprintf(stderr, "%s: [pre-reload] %s\n", __func__, buf);
+                }
+#ifdef _WIN32
+                _pclose(fp);
+#else
+                pclose(fp);
+#endif
+            } else {
+                fprintf(stderr, "%s: failed to execute pre-reload script: %s\n", __func__, pre_script);
+            }
+        }
+
+        if (hotswap_env) {
+            if (!llama_reload_changed_tensors(ctx)) {
+                break;
+            }
+        } else {
+            break;
+        }
+    }
 
     llama_free(ctx);
     llama_free_model(model);

examples/server/server.cpp

@@ -763,6 +763,14 @@ int main(int argc, char ** argv) {
                     }
 
                     res.set_content(health.dump(), "application/json");
+
+                    const char * hotswap_env = std::getenv("LLAMA_HOTSWAP_ENABLED");
+                    if (hotswap_env) {
+                        // WARNING: llama_reload_changed_tensors is NOT thread-safe with active inference.
+                        // Only enable this when you can guarantee the server is idle during health checks.
+                        llama_reload_changed_tensors(ctx_server.ctx);
+                    }
+
                     break;
                 }
             case SERVER_STATE_LOADING_MODEL: