From 7174a124cade15dc93ea00aa87e603e905df346b Mon Sep 17 00:00:00 2001 From: mb8565 <244351746+mb8565@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:40:06 -0500 Subject: [PATCH] CUDA: route P100 (sm_60) decode flash-attention to fp32 vec kernel (#2144) On P100 (GP100, sm_60) the fp16 vec kernel used for decode (batch<=8) accumulates the online-softmax denominator and the P*V product in fp16, flipping ~3-4% of decode top-1 tokens vs an all-fp32 reference (llama.cpp#25593). Decode is memory-bandwidth-bound on P100, so routing sm_60 decode to the in-tree vec_f32 kernel is free (tg128 ~identical). Gated on cc == CC_PASCAL && Q->ne[1] <= 8 (decode only) inside the !fp16_mma_available block, so the prefill tile_f16 path, the D=256 prefill vec path, and fast_fp16_available() are untouched, and the is_pascal_mla_absorbed_decode early-return (MLA) is unaffected. Co-authored-by: Claude Opus 4.8 (1M context) --- ggml/src/ggml-cuda/fattn.cu | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index cc9666be2..4e487e7df 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -96,7 +96,14 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst } if (precision == GGML_PREC_DEFAULT) { if (Q->ne[1] <= 8 || Q->ne[0] == 256) { - ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); + // P100 (sm_60/CC_PASCAL): the fp16 vec kernel accumulates the online-softmax + // denominator and the P*V product in fp16, flipping ~3-4% of decode top-1 tokens + // vs fp32 (llama.cpp#25593). Decode is bandwidth-bound on P100 so vec_f32 is free. + if (cc == CC_PASCAL && Q->ne[1] <= 8) { // decode only (batch<=8); D=256 prefill stays vec_f16 + ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); + } else { + ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); + } } else { ggml_cuda_flash_attn_ext_tile_f16(ctx, dst); } @@ -194,6 +201,9 @@ bool ggml_cuda_fattn_is_supported(ggml_backend_cuda_context & ctx, const ggml_te } if (precision == GGML_PREC_DEFAULT) { if (Q->ne[1] <= 8 || Q->ne[0] == 256) { + if (cc == CC_PASCAL && Q->ne[1] <= 8) { // decode only (batch<=8); D=256 prefill stays vec_f16 + return ggml_cuda_fattn_vec_f32_is_supported(ctx, dst); + } return ggml_cuda_fattn_vec_f16_is_supported(ctx, dst); } else { return ggml_cuda_fattn_tile_f16_is_supported(ctx, dst);