furyhawk/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-07-21 10:15:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Various optimizations

Browse Source
This commit is contained in:
Kawrakow
2026-07-17 12:51:56 +00:00
parent 3d6acd350d
commit ef7d492779
3 changed files with 18 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphs/build_deepseek4.cpp
+9 -3
View File
@@ -530,6 +530,10 @@ static ggml_tensor * dsv4_build_attn(
v = ggml_cast(ctx, v, GGML_TYPE_F16);
}
if (kq_mask->type == GGML_TYPE_F32) {
kq_mask = ggml_cast(ctx, kq_mask, GGML_TYPE_F16);
}
ggml_tensor * cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias,
hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f);
// DSV4 uses the generic CPU FA path here for numerical correctness.
@@ -646,8 +650,9 @@ static ggml_tensor * build_hc_pre(
*comb = ggml_cont(ctx0, dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc));
*comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb);
*comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt);
*comb = build_hc_sinkhorn(ctx0, hparams, *comb);
*comb = ggml_sinkhorn(ctx0, *comb, hc, hparams.dsv4_hc_sinkhorn_iters, hparams.dsv4_hc_eps, false);
//*comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt);
//*comb = build_hc_sinkhorn(ctx0, hparams, *comb);
return llm.build_mhc_weighted_sum(x, pre, n_embd, hc);
}
@@ -703,7 +708,8 @@ static ggml_tensor * build_hca_compressed_kv_from_state(
ggml_tensor * weights = ggml_soft_max(ctx0, scores);
ggml_tensor * comp = ggml_mul(ctx0, values, weights);
comp = ggml_sum_rows(ctx0, comp);
comp = ggml_cont(ctx0, ggml_permute(ctx0, comp, 1, 0, 2, 3));
comp = ggml_reshape_4d(ctx0, comp, comp->ne[1], 1, comp->ne[2], comp->ne[3]);
//comp = ggml_cont(ctx0, ggml_permute(ctx0, comp, 1, 0, 2, 3));
llm.cb(comp, "hca_comp_merge", il);
comp = llm.llm_build_norm(ctx0, comp, llm.hparams, norm, nullptr, LLM_NORM_RMS, llm.cb, il);
src/llama-build-context.cpp
+7 -5
View File
@@ -592,6 +592,10 @@ ggml_tensor * llm_build_context::build_mhc_weighted_sum(
const int64_t n_tokens = x->ne[2];
ggml_tensor * out = nullptr;
if (weights->ne[3] == 1 && x->ne[3] == 1) {
auto w = ggml_reshape_4d(ctx0, weights, 1, weights->ne[0], weights->ne[1], weights->ne[2]);
return ggml_mul_multi_add(ctx0, x, w);
}
for (int64_t stream = 0; stream < n_stream; ++stream) {
ggml_tensor * x_stream = ggml_cont(ctx0,
ggml_view_2d(ctx0, x, n_embd, n_tokens,
@@ -614,15 +618,13 @@ ggml_tensor * llm_build_context::build_mhc_pre_projection(
int64_t n_stream,
float norm_rms_eps,
bool force_contiguous) {
if (force_contiguous) {
if (force_contiguous && !ggml_is_contiguous(x)) {
x = ggml_cont(ctx0, x);
}
ggml_tensor * flat = ggml_reshape_2d(ctx0, x, n_embd*n_stream, x->ne[2]);
ggml_tensor * normed = ggml_rms_norm(ctx0, flat, norm_rms_eps);
if (gamma != nullptr) {
normed = ggml_mul(ctx0, normed, gamma);
}
ggml_tensor * normed = gamma ? ggml_fused_rms_norm(ctx0, flat, gamma, hparams.f_norm_rms_eps) :
ggml_rms_norm(ctx0, flat, norm_rms_eps);
return ggml_mul_mat(ctx0, fn, normed);
}
src/llama-dsv4.cpp
+2 -2
View File
@@ -24,8 +24,8 @@ static bool dsv4_validate_cache_type(ggml_type type, int64_t width, const char *
return false;
}
if (ggml_is_quantized(type) && width % ggml_blck_size(type) != 0) {
LLAMA_LOG_ERROR("%s: DSV4 %s cache width %lld is not aligned to %d elements for %s\n",
__func__, name, (long long) width, ggml_blck_size(type), ggml_type_name(type));
LLAMA_LOG_ERROR("%s: DSV4 %s cache width %d is not aligned to %d elements for %s\n",
__func__, name, (int)width, (int)ggml_blck_size(type), ggml_type_name(type));
return false;
}
return true;