furyhawk/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-07-21 10:15:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/hybrid-cache-speculative

Browse Source
This commit is contained in:
SamuelOliveirads
2026-04-17 13:53:41 -03:00
parent dc4797b723 64234e3c4e
commit 57eabf04df
15 changed files with 485 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/mtmd/clip-impl.h
+12
View File
@@ -80,6 +80,9 @@
#define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm
#define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale
#define TN_LS_2 "%s.blk.%d.ls2.%s" // layer scale
#define TN_LS_OUT "%s.blk.%d.out_scale.%s" // layer out scale (gemma4)
#define TN_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s" // post-attn norm (gemma4)
#define TN_FFN_POST_NORM "%s.blk.%d.ffn_post_norm.%s" // post-FFN norm (gemma4)
#define TN_LN_PRE "%s.pre_ln.%s"
#define TN_LN_POST "%s.post_ln.%s"
#define TN_LLAVA_PROJ "mm.%d.%s"
@@ -122,6 +125,10 @@
#define TN_MM_NORM_PRE "mm.a.norm_pre.%s"
#define TN_MM_NORM_MID "mm.a.norm_mid.%s"
// gemma4
#define TN_STD_BIAS "v.std_bias"
#define TN_STD_SCALE "v.std_scale"
// cogvlm
#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
#define TN_MM_H_TO_4H "mm.up.%s"
@@ -143,6 +150,8 @@ enum projector_type {
PROJECTOR_TYPE_QWEN2VL,
PROJECTOR_TYPE_QWEN3VL,
PROJECTOR_TYPE_GEMMA3,
PROJECTOR_TYPE_GEMMA4V,
PROJECTOR_TYPE_GEMMA4A,
PROJECTOR_TYPE_IDEFICS3,
PROJECTOR_TYPE_PIXTRAL,
PROJECTOR_TYPE_QWEN25VL,
@@ -159,6 +168,7 @@ enum projector_type {
PROJECTOR_TYPE_COGVLM,
PROJECTOR_TYPE_JANUS_PRO,
PROJECTOR_TYPE_UNKNOWN,
};
static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
@@ -171,6 +181,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"},
{ PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"},
{ PROJECTOR_TYPE_GEMMA3, "gemma3"},
{ PROJECTOR_TYPE_GEMMA4V, "gemma4v"},
{ PROJECTOR_TYPE_GEMMA4A, "gemma4a"},
{ PROJECTOR_TYPE_IDEFICS3, "idefics3"},
{ PROJECTOR_TYPE_PIXTRAL, "pixtral"},
{ PROJECTOR_TYPE_ULTRAVOX, "ultravox"},
examples/mtmd/clip.cpp
+311 -10
View File
@@ -34,6 +34,7 @@
#include <limits>
#include <array>
#include <functional>
#include <cfloat>
#define DEFAULT_INTERPOLATION_MODE ((int)GGML_SCALE_MODE_BILINEAR | (int)GGML_SCALE_FLAG_ALIGN_CORNERS)
@@ -162,6 +163,10 @@ static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u
}
#endif
static inline bool string_ends_with(std::string_view str, std::string_view suffix) {
return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
}
//
// clip layers
@@ -273,6 +278,10 @@ struct clip_layer {
// layer scale (no bias)
ggml_tensor * ls_1_w = nullptr;
ggml_tensor * ls_2_w = nullptr;
ggml_tensor * ls_out_w = nullptr; // gemma4
ggml_tensor * ff_post_norm_w = nullptr;
ggml_tensor * attn_post_norm_w = nullptr;
// qwen3vl deepstack merger
ggml_tensor * deepstack_norm_w = nullptr;
@@ -414,6 +423,18 @@ struct clip_model {
ggml_tensor * mm_boi = nullptr;
ggml_tensor * mm_eoi = nullptr;
// gemma4
ggml_tensor * std_bias = nullptr;
ggml_tensor * std_scale = nullptr;
// Gemma4ClippableLinear
struct clamp_info {
float inp_max;
float inp_min;
float out_max;
float out_min;
};
std::map<std::string, clamp_info> clamp_info_map;
bool audio_has_avgpool() const {
return proj_type == PROJECTOR_TYPE_QWEN2A
|| proj_type == PROJECTOR_TYPE_VOXTRAL;
@@ -536,7 +557,7 @@ struct clip_graph {
const int d_head;
const int n_layer;
const float eps;
const float kq_scale;
float kq_scale;
ggml_context_ptr ctx0_ptr;
ggml_context * ctx0;
@@ -1117,6 +1138,156 @@ struct clip_graph {
return gf;
}
ggml_tensor * build_mm_gemma4(ggml_tensor * w, ggml_tensor * x) const {
// Gemma4ClippableLinear
auto it = model.clamp_info_map.find(w->name);
if (it == model.clamp_info_map.end()) {
return ggml_mul_mat(ctx0, w, x);
} else {
const auto & clamp_info = it->second;
ggml_tensor * clamped = ggml_clamp(ctx0, x, clamp_info.inp_min, clamp_info.inp_max);
ggml_tensor * out = ggml_mul_mat(ctx0, w, clamped);
out = ggml_clamp(ctx0, out, clamp_info.out_min, clamp_info.out_max);
return out;
}
}
// Gemma4
ggml_cgraph * build_gemma4() {
ggml_tensor * inp_raw = build_inp_raw();
// patches = 2 * (patches - 0.5)
// equivalent to: patches * 2 - 1
inp_raw = ggml_scale_bias(ctx0, inp_raw, 2.0f, -1.0f);
ggml_set_name(inp_raw, "inp_raw_scaled");
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd);
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
ggml_set_name(inp, "inp");
// note: no patch bias
ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_x, "pos_x");
ggml_set_input(pos_x);
ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_y, "pos_y");
ggml_set_input(pos_y);
{
const int64_t pos_size = model.position_embeddings->ne[1];
const size_t nb1 = ggml_row_size(model.position_embeddings->type, n_embd);
// positional embeddings are stored as lookup tables (one for x, one for y)
ggml_tensor * tbl_x = ggml_view_2d(ctx0, model.position_embeddings,
n_embd, pos_size, nb1, 0);
ggml_tensor * tbl_y = ggml_view_2d(ctx0, model.position_embeddings,
n_embd, pos_size, nb1, pos_size * nb1);
// ggml_get_rows: [n_embd, n_patches]
ggml_tensor * emb_x = ggml_get_rows(ctx0, tbl_x, pos_x);
ggml_tensor * emb_y = ggml_get_rows(ctx0, tbl_y, pos_y);
inp = ggml_add(ctx0, inp, emb_x);
inp = ggml_add(ctx0, inp, emb_y);
cb(inp, "pos_embd", -1);
}
// similar to build_rope_2d, but use neox ordering
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
const int64_t n_dim = cur->ne[0];
const int64_t n_head = cur->ne[1];
const int64_t n_pos = cur->ne[2];
// first half
ggml_tensor * first;
{
first = ggml_view_3d(ctx0, cur,
n_dim/2, n_head, n_pos,
cur->nb[1],
cur->nb[2],
0);
first = ggml_rope_ext(
ctx0,
first,
pos_x, // positions
nullptr, // freq factors
n_dim/2, // n_dims
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
1.0f, 0.0f, 1.0f, 0.0f, 0.0f
);
}
// second half
ggml_tensor * second;
{
second = ggml_view_3d(ctx0, cur,
n_dim/2, n_head, n_pos,
cur->nb[1],
cur->nb[2],
n_dim/2 * ggml_element_size(cur));
second = ggml_rope_ext(
ctx0,
second,
pos_y, // positions
nullptr, // freq factors
n_dim/2, // n_dims
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
1.0f, 0.0f, 1.0f, 0.0f, 0.0f
);
}
cur = ggml_concat(ctx0, first, second, 0);
return cur;
};
kq_scale = 1.0f;
ggml_tensor * cur = build_vit(
inp, n_patches,
NORM_TYPE_RMS,
hparams.ffn_op,
nullptr, // pos embd is already handled above
add_pos);
// Gemma4VisionPooler
{
const int kernel_size = hparams.n_merge;
GGML_ASSERT(kernel_size > 0);
// [n_embd, n_patches] -> [n_patches_x, n_patches_y, n_embd, 1]
cur = ggml_cont_4d(ctx0, ggml_transpose(ctx0, cur), n_patches_x, n_patches_y, n_embd, 1);
cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG,
kernel_size, kernel_size, kernel_size, kernel_size, 0, 0);
const int out_x = n_patches_x / kernel_size;
const int out_y = n_patches_y / kernel_size;
// [out_x, out_y, n_embd, 1] -> [n_embd, out_x * out_y]
cur = ggml_reshape_3d(ctx0, cur, out_x * out_y, n_embd, 1);
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cur = ggml_scale(ctx0, cur, sqrtf((float)n_embd));
cb(cur, "pooled", -1);
}
// hidden_states = (hidden_states - self.std_bias) * self.std_scale
if (model.std_bias && model.std_scale) {
cur = ggml_sub(ctx0, cur, model.std_bias);
cur = ggml_mul(ctx0, cur, model.std_scale);
cb(cur, "std_scaled", -1);
}
// Gemma4MultimodalEmbedder
cur = build_mm_gemma4(model.mm_input_proj_w, cur);
cb(cur, "projected", -1);
// embedding_post_projection_norm
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
cb(cur, "projected_normed", -1);
ggml_build_forward_expand(gf, cur);
return gf;
}
ggml_cgraph * build_minicpmv() {
const int batch_size = 1;
@@ -2164,6 +2335,17 @@ private:
// TODO: q/k norm requires row size == n_embd, while here it's d_head
// we can add support in the future if needed
GGML_ASSERT(layer.q_norm == nullptr && layer.k_norm == nullptr);
if (layer.q_norm) {
GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
cb(Qcur, "Qcur_norm", il);
}
if (layer.k_norm) {
GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);
Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
cb(Kcur, "Kcur_norm", il);
}
} else {
// separate q, k, v
@@ -2182,19 +2364,35 @@ private:
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
}
if (layer.q_norm) {
Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
cb(Qcur, "Qcur_norm", il);
}
bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head;
if (layer.k_norm) {
Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
cb(Kcur, "Kcur_norm", il);
if (!norm_per_head) {
if (layer.q_norm) {
Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
cb(Qcur, "Qcur_norm", il);
}
if (layer.k_norm) {
Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
cb(Kcur, "Kcur_norm", il);
}
}
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
if (norm_per_head) {
if (layer.q_norm) {
Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
cb(Qcur, "Qcur_norm_per_head", il);
}
if (layer.k_norm) {
Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
cb(Kcur, "Kcur_norm_per_head", il);
}
}
}
cb(Qcur, "Qcur", il);
@@ -2208,6 +2406,11 @@ private:
cb(Kcur, "Kcur_pos", il);
}
if (model.proj_type == PROJECTOR_TYPE_GEMMA4V) {
Vcur = ggml_rms_norm(ctx0, Vcur, eps);
cb(Vcur, "Vcur_normed", il);
}
cur = build_attn(layer.o_w, layer.o_b,
Qcur, Kcur, Vcur, nullptr, kq_scale, il);
cb(cur, "attn_out", il);
@@ -2218,6 +2421,11 @@ private:
cb(cur, "attn_out_scaled", il);
}
if (layer.attn_post_norm_w) {
cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il);
cb(cur, "attn_post_normed", il);
}
// re-add the layer input, e.g., residual
cur = ggml_add(ctx0, cur, inpL);
@@ -2238,6 +2446,11 @@ private:
cb(cur, "ffn_out", il);
if (layer.ff_post_norm_w) {
cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il);
cb(cur, "ffn_post_normed", il);
}
if (layer.ls_2_w) {
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
cb(cur, "ffn_out_scaled", il);
@@ -2247,6 +2460,11 @@ private:
cur = ggml_add(ctx0, inpL, cur);
cb(cur, "layer_out", il);
if (layer.ls_out_w) {
cur = ggml_mul(ctx0, cur, layer.ls_out_w);
cb(cur, "layer_out_scaled", il);
}
inpL = cur;
}
@@ -2658,6 +2876,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
res = graph.build_qwen3vl();
} break;
case PROJECTOR_TYPE_GEMMA4V:
{
res = graph.build_gemma4();
} break;
case PROJECTOR_TYPE_MINICPMV:
{
res = graph.build_minicpmv();
@@ -3018,6 +3240,16 @@ struct clip_model_loader {
LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
}
} break;
case PROJECTOR_TYPE_GEMMA4V:
{
hparams.rope_theta = 100.0f;
hparams.n_merge = 3; // pooling_kernel_size
//hparams.image_resize_algo = img_tool::RESIZE_ALGO_BILINEAR;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
// @ngxson : the model performs quite poor with small images, we need to bump minimum image tokens to 40 to avoid that
hparams.set_limit_image_tokens(252, 280);
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_LLAMA4:
{
hparams.rope_theta = 10000.0f;
@@ -3086,6 +3318,11 @@ struct clip_model_loader {
std::map<std::string, size_t> tensor_offset;
std::vector<ggml_tensor *> tensors_to_load;
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str()));
}
// TODO @ngxson : support both audio and video in the future
const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" : "v";
@@ -3122,6 +3359,19 @@ struct clip_model_loader {
return cur;
};
auto get_scalar = [&](const std::string & name, float default_val) {
auto it = tensor_offset.find(name);
if (it == tensor_offset.end()) {
return default_val;
}
size_t offset = it->second;
fin.seekg(offset, std::ios::beg);
float value;
fin.read(reinterpret_cast<char*>(&value), sizeof(float));
return value;
};
model.class_embedding = get_tensor(TN_CLASS_EMBD, false);
model.pre_ln_w = get_tensor(string_format(TN_LN_PRE, prefix, "weight"), false);
@@ -3151,6 +3401,9 @@ struct clip_model_loader {
layer.ln_2_w = get_tensor(string_format(TN_LN_2, prefix, il, "weight"), false);
layer.ls_1_w = get_tensor(string_format(TN_LS_1, prefix, il, "weight"), false); // no bias
layer.ls_2_w = get_tensor(string_format(TN_LS_2, prefix, il, "weight"), false); // no bias
layer.ls_out_w = get_tensor(string_format(TN_LS_OUT, prefix, il, "weight"), false); // no bias
layer.attn_post_norm_w = get_tensor(string_format(TN_ATTN_POST_NORM, prefix, il, "weight"), false); // no bias
layer.ff_post_norm_w = get_tensor(string_format(TN_FFN_POST_NORM, prefix, il, "weight"), false); // no bias
layer.k_b = get_tensor(string_format(TN_ATTN_K, prefix, il, "bias"), false);
layer.q_b = get_tensor(string_format(TN_ATTN_Q, prefix, il, "bias"), false);
@@ -3327,6 +3580,32 @@ struct clip_model_loader {
model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N);
} break;
case PROJECTOR_TYPE_GEMMA4V:
{
model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
model.std_bias = get_tensor(TN_STD_BIAS, false);
model.std_scale = get_tensor(TN_STD_SCALE, false);
// load scalar for Gemma4ClippableLinear
for (auto * tensor : tensors_to_load) {
std::string name = tensor->name;
if (string_ends_with(name, ".weight")) {
std::string name_inp_max = name;
std::string name_inp_min = name;
std::string name_out_max = name;
std::string name_out_min = name;
string_replace_all(name_inp_max, ".weight", ".input_max");
string_replace_all(name_inp_min, ".weight", ".input_min");
string_replace_all(name_out_max, ".weight", ".output_max");
string_replace_all(name_out_min, ".weight", ".output_min");
model.clamp_info_map[name] = {
get_scalar(name_inp_max, FLT_MAX),
get_scalar(name_inp_min, -FLT_MAX),
get_scalar(name_out_max, FLT_MAX),
get_scalar(name_out_min, -FLT_MAX)
};
}
}
} break;
case PROJECTOR_TYPE_IDEFICS3:
{
model.projection = get_tensor(TN_MM_PROJECTOR);
@@ -3703,14 +3982,18 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
try {
clip_model_loader loader(fname);
bool skip_audio = false;
if (loader.has_vision) {
ctx_vision = new clip_ctx(ctx_params);
loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
loader.load_tensors(*ctx_vision);
loader.warmup(*ctx_vision);
if (ctx_vision->model.proj_type == PROJECTOR_TYPE_GEMMA4V) {
skip_audio = true;
}
}
if (loader.has_audio) {
if (!skip_audio && loader.has_audio) {
ctx_audio = new clip_ctx(ctx_params);
loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO);
loader.load_tensors(*ctx_audio);
@@ -4415,12 +4698,14 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_GEMMA4V:
{
GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0);
clip_image_u8 resized;
const int cur_merge = params.n_merge == 0 ? 1 : params.n_merge;
const clip_image_size new_size = img_tool::calc_size_preserved_ratio(
original_size,
params.patch_size * 2,
params.patch_size * cur_merge,
params.image_min_pixels,
params.image_max_pixels);
img_tool::resize(*img, resized, new_size, img_tool::RESIZE_ALGO_BILINEAR, false);
@@ -4749,6 +5034,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_LLAMA4:
case PROJECTOR_TYPE_GEMMA4V:
{
// both X and Y are downscaled by the scale factor
int scale_factor = ctx->model.hparams.n_merge;
@@ -5073,6 +5359,19 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_GEMMA4V:
{
// set (col, row) patch positions for learned positional embedding
// what about image_size_height?
const int n_cols = image_size_width / patch_size;
std::vector<int> pos_x(num_patches), pos_y(num_patches);
for (int i = 0; i < num_patches; i++) {
pos_x[i] = i % n_cols;
pos_y[i] = i / n_cols;
}
set_input_i32("pos_x", pos_x);
set_input_i32("pos_y", pos_y);
} break;
case PROJECTOR_TYPE_QWEN25VL:
{
// pw * ph = number of tokens output by ViT after apply patch merger
@@ -5300,6 +5599,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);
case PROJECTOR_TYPE_GEMMA3:
return ctx->model.mm_input_proj_w->ne[0];
case PROJECTOR_TYPE_GEMMA4V:
return ctx->model.mm_input_proj_w->ne[1];
case PROJECTOR_TYPE_IDEFICS3:
return ctx->model.projection->ne[1];
case PROJECTOR_TYPE_ULTRAVOX:
examples/mtmd/mtmd-cli.cpp
+4 -4
View File
@@ -252,11 +252,11 @@ static int generate_response(mtmd_cli_context & ctx, int n_predict) {
return 0;
}
static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, bool add_bos = false) {
static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, bool add_bos = false, bool use_jinja = false) {
common_chat_templates_inputs tmpl_inputs;
tmpl_inputs.messages = {msg};
tmpl_inputs.add_generation_prompt = true;
tmpl_inputs.use_jinja = false; // jinja is buggy here
tmpl_inputs.use_jinja = use_jinja; // jinja is buggy here
auto formatted_chat = common_chat_templates_apply(ctx.tmpls.get(), tmpl_inputs);
LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.prompt.c_str());
@@ -362,7 +362,7 @@ int main(int argc, char ** argv) {
return 1; // error is already printed by libmtmd
}
}
if (eval_message(ctx, msg, true)) {
if (eval_message(ctx, msg, true, params.use_jinja)) {
return 1;
}
if (!g_is_interrupted && generate_response(ctx, n_predict)) {
@@ -427,7 +427,7 @@ int main(int argc, char ** argv) {
common_chat_msg msg;
msg.role = "user";
msg.content = content;
int ret = eval_message(ctx, msg, is_first_msg);
int ret = eval_message(ctx, msg, is_first_msg, params.use_jinja);
if (ret) {
return 1;
}
examples/mtmd/mtmd.cpp
+10 -2
View File
@@ -301,6 +301,12 @@ struct mtmd_context {
img_end = "<|im_end|>";
}
else if (proj == PROJECTOR_TYPE_GEMMA4V) {
// <|image> ... (image embeddings) ... <image|>
img_beg = "<|image>";
img_end = "<image|>";
//image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
}
}
void init_audio() {
@@ -831,8 +837,10 @@ float * mtmd_get_output_embd(mtmd_context * ctx) {
}
bool mtmd_decode_use_non_causal(mtmd_context * ctx) {
if (ctx->ctx_v && clip_get_projector_type(ctx->ctx_v) == PROJECTOR_TYPE_GEMMA3) {
return true;
if (ctx->ctx_v) {
if (auto type = clip_get_projector_type(ctx->ctx_v); type == PROJECTOR_TYPE_GEMMA3 || type == PROJECTOR_TYPE_GEMMA4V) {
return true;
}
}
return false;
}
examples/server/server-context.cpp
+18 -8
View File
@@ -114,6 +114,15 @@ bool server_context::load_model(const gpt_params& params_) {
}
// Load draft model for speculative decoding if specified
if (has_draft_model) {
if (llama_model_has_recurrent(model)) {
LLAMA_LOG_WARN("\n=======================================================================\n");
LLAMA_LOG_WARN(" Speculative decodong is not suported for recurrent/hybrid models\n");
LLAMA_LOG_WARN(" --> bailing out\n");
LLAMA_LOG_WARN("========================================================================\n\n");
GGML_ABORT("Fatal error");
}
LLAMA_LOG_INFO("\n\n==================================loading DRAFT model==================================\n\n");
gpt_params params_dft;
@@ -1478,9 +1487,9 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task)
//
// TODO: try to make this conditional on the context or the memory module, instead of the model type
params_base.do_checkpoint = do_checkpoint;
if (slot.n_buffer != 0) {
LLAMA_LOG_WARN("banned strings is not supported by recurrent model, it will be disabled.\n");
}
if (slot.n_buffer != 0) {
LLAMA_LOG_WARN("banned strings is not supported by recurrent model, it will be disabled.\n");
}
if (params_base.ctx_shift) {
params_base.ctx_shift = false;
LOG_WARNING("%s\n", "ctx_shift is not supported by recurrent model, it will be disabled");
@@ -1976,6 +1985,7 @@ void server_context::send_final_response(server_slot& slot) {
res->oai_resp_reasoning_id = slot.oai_resp_reasoning_id;
res->oai_resp_message_id = slot.oai_resp_message_id;
res->n_decoded = slot.n_decoded;
res->n_prompt_tokens_cache = slot.n_prompt_tokens_cache;
res->anthropic_thinking_block_started = slot.anthropic_thinking_block_started;
res->anthropic_text_block_started = slot.anthropic_text_block_started;
res->n_prompt_tokens = slot.n_prompt_tokens;
@@ -3320,16 +3330,16 @@ void server_context::batch_pending_prompt(const int32_t n_ubatch, const int32_t
LLAMA_LOG_INFO("======== Cache: cache_size = %d, n_past0 = %d, n_past1 = %d, n_past_prompt1 = %d, n_past2 = %d, n_past_prompt2 = %d\n", (int32_t)slot.cache_tokens.size(), (int32_t)n_past0, (int32_t)prefix.first, (int32_t)prefix.second, (int32_t)prefix_nonexact.first, (int32_t)prefix_nonexact.second);
int32_t size_threshold = 20;
if (prefix.first + size_threshold < prefix_nonexact.first) {
LLAMA_LOG_WARN("Common part contains missing or extra space and new line\n");
// LLAMA_LOG_WARN("Common part contains missing or extra space and new line\n");
prefix = prefix_nonexact;
}
slot.n_past = prefix.first;
slot.n_past_prompt = prefix.second;
slot.n_past_offset = slot.n_past_prompt - slot.n_past;
if (slot.n_past != slot.n_past_prompt) {
LLAMA_LOG_INFO("Mistokenization found and handled successfully.\n");
}
//if (slot.n_past != slot.n_past_prompt) {
// LLAMA_LOG_INFO("Mistokenization found and handled successfully.\n");
//}
if ((slot.n_past + size_threshold < slot.cache_tokens.size()))
{
LLAMA_LOG_WARN("Common part does not match fully\n");
@@ -3359,7 +3369,7 @@ void server_context::batch_pending_prompt(const int32_t n_ubatch, const int32_t
slot.n_past_se--;
}
}
slot.n_prompt_tokens_cache = slot.n_past_prompt;
slot.n_prompt_tokens_processed = 0;
}
examples/server/server-context.h
+1
View File
@@ -56,6 +56,7 @@ struct server_slot {
int32_t n_predict = -1; // TODO: disambiguate from params.n_predict
int32_t n_prompt_tokens = 0;
int32_t n_prompt_tokens_cache = 0;
int32_t n_prompt_tokens_processed = 0;
json prompt; // can be either a string, array of strings or array of token ids
examples/server/server-task.cpp
+22 -20
View File
@@ -120,6 +120,16 @@ json server_task_result_cmpl_partial::to_json_oaicompat_partial() {
return res;
}
json server_task_result_cmpl_final::usage_json_oaicompat() {
return json{
{"completion_tokens", n_decoded},
{"prompt_tokens", n_prompt_tokens},
{"total_tokens", n_decoded + n_prompt_tokens},
{"prompt_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},
};
}
json server_task_result_cmpl_final::to_json_oaicompat_final() {
std::time_t t = std::time(0);
json logprobs = json(nullptr); // OAI default to null
@@ -144,11 +154,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_final() {
{"created", t},
{"model", oaicompat_model},
{"object", "text_completion"},
{"usage", json {
{"completion_tokens", n_decoded},
{"prompt_tokens", n_prompt_tokens},
{"total_tokens", n_decoded + n_prompt_tokens}
}},
{"usage", usage_json_oaicompat()},
{"id", oaicompat_cmpl_id}
};
@@ -379,11 +385,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_final() {
{"created", t},
{"model", oaicompat_model},
{"object", "chat.completion"},
{"usage", json {
{"completion_tokens", n_decoded},
{"prompt_tokens", n_prompt_tokens},
{"total_tokens", n_decoded + n_prompt_tokens}
}},
{"usage", usage_json_oaicompat()},
{"id", oaicompat_cmpl_id}
};
@@ -445,11 +447,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
{"id", oaicompat_cmpl_id},
{"model", oaicompat_model},
{"object", "chat.completion.chunk"},
{"usage", json {
{"completion_tokens", n_decoded},
{"prompt_tokens", n_prompt_tokens},
{"total_tokens", n_decoded + n_prompt_tokens},
}},
{"usage", usage_json_oaicompat()},
});
}
if (timings.prompt_n >= 0) {
@@ -523,10 +521,11 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_final() {
{"object", "response"},
{"output", output},
{"status", "completed"},
{"usage", json{
{"usage", json {
{"input_tokens", n_prompt_tokens},
{"output_tokens", n_decoded},
{"total_tokens", n_decoded + n_prompt_tokens},
{"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},
}},
};
@@ -633,11 +632,12 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
{"status", "completed"},
{"model", oaicompat_model},
{"output", output},
{"usage", json{
{"usage", json {
{"input_tokens", n_prompt_tokens},
{"output_tokens", n_decoded},
{"total_tokens", n_decoded + n_prompt_tokens},
}},
{"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},
}}
}},
}},
});
@@ -703,7 +703,8 @@ json server_task_result_cmpl_final::to_json_anthropic_final() {
{"stop_reason", stop_reason},
{"stop_sequence", stopping_word.empty() ? nullptr : json(stopping_word)},
{"usage", {
{"input_tokens", n_prompt_tokens},
{"cache_read_input_tokens", n_prompt_tokens_cache},
{"input_tokens", n_prompt_tokens - n_prompt_tokens_cache},
{"output_tokens", n_decoded}
}}
};
@@ -923,7 +924,8 @@ json server_task_result_cmpl_partial::to_json_anthropic_partial() {
{"stop_reason", nullptr},
{"stop_sequence", nullptr},
{"usage", {
{"input_tokens", n_prompt_tokens},
{"cache_read_input_tokens", n_prompt_tokens_cache},
{"input_tokens", n_prompt_tokens - n_prompt_tokens_cache},
{"output_tokens", 0}
}}
}}
examples/server/server-task.h
+3
View File
@@ -166,6 +166,7 @@ struct server_task_result {
bool truncated;
int32_t n_decoded;
int32_t n_prompt_tokens;
int32_t n_prompt_tokens_cache;
int32_t n_tokens_cached;
bool has_new_line;
std::string stopping_word;
@@ -258,6 +259,8 @@ struct server_task_result_cmpl_final : server_task_result {
json to_json_non_oaicompat_final();
json usage_json_oaicompat();
json to_json_oaicompat_final();
json to_json_oaicompat_chat_final();
examples/server/server.cpp
+2 -1
View File
@@ -1262,7 +1262,8 @@ int main(int argc, char ** argv) {
{"object", "model"},
{"created", std::time(0)},
{"owned_by", "llamacpp"},
{"meta", model_meta}
{"meta", model_meta},
{"max_model_len", params.n_ctx}, //vllm specs
},
}}
};
ggml/src/ggml-cuda.cu
+13
View File
@@ -4454,6 +4454,19 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons
return false;
}
break;
case GGML_OP_GLU:
switch (ggml_get_glu_op(op)) {
case GGML_GLU_OP_REGLU:
case GGML_GLU_OP_GEGLU:
case GGML_GLU_OP_SWIGLU:
case GGML_GLU_OP_SWIGLU_OAI:
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
return ggml_is_contiguous_1(op->src[0]);
default:
return false;
}
break;
case GGML_OP_FUSED_MUL_UNARY: return ggml_is_contiguous(op->src[0]);
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
ggml/src/ggml-cuda/norm.cu
+2 -1
View File
@@ -438,7 +438,8 @@ static void group_norm_f32_cuda(const float * x, float * dst, const int num_grou
}
static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
GGML_ASSERT(ncols % WARP_SIZE == 0);
// Why did we have this assert?
//GGML_ASSERT(ncols % WARP_SIZE == 0);
constexpr int kBlockSize = 256;
if (ncols < 1024) {
const dim3 block_dims(kBlockSize, 1, 1);
ggml/src/ggml.c
+24 -17
View File
@@ -6439,7 +6439,7 @@ static struct ggml_tensor * ggml_sub_impl(
struct ggml_tensor * a,
struct ggml_tensor * b,
bool inplace) {
GGML_ASSERT(ggml_are_same_shape(a, b));
GGML_ASSERT(ggml_can_repeat(b, a));
bool is_node = false;
@@ -13637,7 +13637,8 @@ static void ggml_compute_forward_sub_f32(
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
GGML_ASSERT(src0->ne[0] == src1->ne[0]);
int ith = params->ith;
int nth = params->nth;
@@ -13655,21 +13656,24 @@ static void ggml_compute_forward_sub_f32(
if (nb10 == sizeof(float)) {
for (int ir = first; ir < last; ++ir) {
// src0, src1 and dst are same shape => same indices
const int i3 = ir/(ne2*ne1);
const int i2 = (ir - i3*ne2*ne1)/ne1;
const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
const int i03 = ir/(ne2*ne1);
const int i02 = (ir - i03*ne2*ne1)/ne1;
const int i01 = (ir - i03*ne2*ne1 - i02*ne1);
const int64_t i13 = i03 % ne13;
const int64_t i12 = i02 % ne12;
const int64_t i11 = i01 % ne11;
#ifdef GGML_USE_ACCELERATE
vDSP_vsub(
(float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11), 1,
(float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), 1,
(float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), 1,
(float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), 1,
(float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01), 1,
(float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ), 1,
ne0);
#else
ggml_vec_sub_f32(ne0,
(float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),
(float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),
(float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));
(float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ),
(float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01),
(float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11));
#endif
// }
// }
@@ -13678,13 +13682,16 @@ static void ggml_compute_forward_sub_f32(
// src1 is not contiguous
for (int ir = 0; ir < nr; ++ir) {
// src0, src1 and dst are same shape => same indices
const int i3 = ir/(ne2*ne1);
const int i2 = (ir - i3*ne2*ne1)/ne1;
const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
const int i03 = ir/(ne2*ne1);
const int i02 = (ir - i03*ne2*ne1)/ne1;
const int i01 = (ir - i03*ne2*ne1 - i02*ne1);
const int64_t i13 = i03 % ne13;
const int64_t i12 = i02 % ne12;
const int64_t i11 = i01 % ne11;
float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
float * src0_ptr = (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
char * src1_ptr = (char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11;
float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );
float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
char * src1_ptr = (char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11;
for (int i0 = 0; i0 < ne0; i0++) {
dst_ptr[i0] = src0_ptr[i0] - *(float *)(src1_ptr + i0*nb10);
}
src/llama-load-tensors.cpp
+1 -1
View File
@@ -312,7 +312,7 @@ create_tensors_helper::create_tensors_helper(llama_model_loader & _ml, llama_mod
// Split MTP layer's to graph
if ((model.split_mode == LLAMA_SPLIT_MODE_GRAPH || model.split_mode == LLAMA_SPLIT_MODE_ATTN) &&
model.hparams.nextn_predict_layers > 0 && model.splits.size() > 1) {
int mtp_first = n_layer - model.hparams.nextn_predict_layers;
[[maybe_unused]] int mtp_first = n_layer - model.hparams.nextn_predict_layers;
LLAMA_LOG_DEBUG("%s: MTP layer(s) %d-%d: split attention+FFN, nextn on per-device CUDA\n",
__func__, mtp_first, n_layer - 1);
}
src/llama-model-loader.cpp
+4 -4
View File
@@ -724,8 +724,8 @@ bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & resul
result.resize(arr_info.length);
if (arr_info.gt == GGUF_TYPE_BOOL) {
std::transform((const bool *)arr_info.data, (const bool *)arr_info.data + arr_info.length, result.begin(),
[] (bool x) { return static_cast<T>(x); });
std::transform((const int8_t *)arr_info.data, (const int8_t *)arr_info.data + arr_info.length, result.begin(),
[] (int8_t x) { return static_cast<T>(x != 0); });
} else {
result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);
@@ -762,8 +762,8 @@ bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> &
}
if (arr_info.gt == GGUF_TYPE_BOOL) {
std::transform((const bool *)arr_info.data, (const bool *)arr_info.data + arr_info.length, result.begin(),
[] (bool x) { return static_cast<T>(x); });
std::transform((const int8_t *)arr_info.data, (const int8_t *)arr_info.data + arr_info.length, result.begin(),
[] (int8_t x) { return static_cast<T>(x != 0); });
} else {
std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());
}
src/llama.cpp
+58 -5
View File
@@ -3598,7 +3598,28 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {
GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));
float * data = (float *) lctx.inp_KQ_mask->data;
float * data = nullptr;
float * data_swa = nullptr;
ggml_half * data_f16 = nullptr;
ggml_half * data_swa_f16 = nullptr;
if (lctx.inp_KQ_mask) {
GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));
if (cparams.flash_attn) {
data_f16 = (ggml_half *)lctx.inp_KQ_mask->data;
} else {
data = (float *) lctx.inp_KQ_mask->data;
}
}
if (lctx.inp_KQ_mask_swa) {
GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask_swa->buffer));
if (cparams.flash_attn) {
data_swa_f16 = (ggml_half *) lctx.inp_KQ_mask_swa->data;
} else {
data_swa = (float *) lctx.inp_KQ_mask_swa->data;
}
}
for (int h = 0; h < 1; ++h) {
for (int j = 0; j < n_tokens; ++j) {
@@ -3617,11 +3638,43 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {
}
}
data[h*(n_tokens*n_tokens) + j*n_stride + i] = f;
if (data) {
data[h*(n_tokens*n_tokens) + j*n_stride + i] = f;
}
if (data_f16) {
data_f16[h*(n_tokens*n_tokens) + j*n_tokens + i] = ggml_fp32_to_fp16(f);
}
if (data_swa || data_swa_f16) {
if (hparams.n_attn_chunk) {
llama_pos pos_chunk_start = (batch.pos[i] / hparams.n_attn_chunk) * hparams.n_attn_chunk;
if (lctx.kv_self.cells[i].pos < pos_chunk_start || batch.pos[i] < pos_chunk_start) {
f = -INFINITY;
}
} else {
if (batch.pos[i] - kv_self.cells[i].pos >= (int32_t)hparams.n_swa) {
f = -INFINITY;
}
}
if (data_swa) {
data_swa[h*(n_tokens*n_tokens) + j*n_tokens + i] = f;
}
if (data_swa_f16) {
data_swa_f16[h*(n_tokens*n_tokens) + j*n_tokens + i] = ggml_fp32_to_fp16(f);
}
}
}
for (int i = n_tokens; i < n_stride; ++i) {
data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY;
if (data) {
for (int i = n_tokens; i < n_stride; ++i) {
data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY;
}
}
if (data_f16) {
auto h_inf = ggml_fp32_to_fp16(-INFINITY);
for (int i = n_tokens; i < n_stride; ++i) {
data_f16[h*(n_tokens*n_tokens) + j*n_stride + i] = h_inf;
}
}
}
}
@@ -4331,7 +4384,7 @@ static int llama_decode_internal(
if (n_outputs_new) {
GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs);
GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_vocab <= (int64_t) lctx.logits_size);
if (res->ne[1] == n_tokens && n_outputs_new < n_tokens) {
int32_t i_out = 0;
if (u_batch.logits && !embd_pooled) {