From 537ce13df79d80a6fb4f54105e688bfacfcaa9d3 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Thu, 16 Jul 2026 08:10:30 +0200 Subject: [PATCH] fix cm2 bindings --- .../vulkan-shaders/mul_mm_cm2.comp | 111 +++++++++++------- 1 file changed, 66 insertions(+), 45 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index ef570d90a2..d10006c95f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -93,7 +93,28 @@ layout (push_constant) uniform parameter #ifndef MULMAT_QUANT layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; #else -layout (binding = 0) readonly buffer A {uint8_t data_a[];}; +layout (binding = 0) readonly buffer BUF_Q1_0 { block_q1_0 data[]; } a_q1_0; +layout (binding = 0) readonly buffer BUF_Q4_0 { block_q4_0 data[]; } a_q4_0; +layout (binding = 0) readonly buffer BUF_Q4_1 { block_q4_1 data[]; } a_q4_1; +layout (binding = 0) readonly buffer BUF_Q5_0 { block_q5_0 data[]; } a_q5_0; +layout (binding = 0) readonly buffer BUF_Q5_1 { block_q5_1 data[]; } a_q5_1; +layout (binding = 0) readonly buffer BUF_Q8_0 { block_q8_0 data[]; } a_q8_0; +layout (binding = 0) readonly buffer BUF_Q2_K { block_q2_K data[]; } a_q2_k; +layout (binding = 0) readonly buffer BUF_Q3_K { block_q3_K data[]; } a_q3_k; +layout (binding = 0) readonly buffer BUF_Q4_K { block_q4_K data[]; } a_q4_k; +layout (binding = 0) readonly buffer BUF_Q5_K { block_q5_K data[]; } a_q5_k; +layout (binding = 0) readonly buffer BUF_Q6_K { block_q6_K data[]; } a_q6_k; +layout (binding = 0) readonly buffer BUF_IQ1_S { block_iq1_s data[]; } a_iq1_s; +layout (binding = 0) readonly buffer BUF_IQ1_M { block_iq1_m data[]; } a_iq1_m; +layout (binding = 0) readonly buffer BUF_IQ2_XXS { block_iq2_xxs data[]; } a_iq2_xxs; +layout (binding = 0) readonly buffer BUF_IQ2_XS { block_iq2_xs data[]; } a_iq2_xs; +layout (binding = 0) readonly buffer BUF_IQ2_S { block_iq2_s data[]; } a_iq2_s; +layout (binding = 0) readonly buffer BUF_IQ3_XXS { block_iq3_xxs data[]; } a_iq3_xxs; +layout (binding = 0) readonly buffer BUF_IQ3_S { block_iq3_s data[]; } a_iq3_s; +layout (binding = 0) readonly buffer BUF_IQ4_XS { block_iq4_xs data[]; } a_iq4_xs; +layout (binding = 0) readonly buffer BUF_IQ4_NL { block_iq4_nl data[]; } a_iq4_nl; +layout (binding = 0) readonly buffer BUF_MXFP4 { block_mxfp4 data[]; } a_mxfp4; +layout (binding = 0) readonly buffer BUF_NVFP4 { block_nvfp4 data[]; } a_nvfp4; #endif layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; @@ -116,52 +137,52 @@ layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];}; // all but the selected branch. #ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR #define COOPMAT_LOAD_A(mat, tensor_layout) \ - if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ1_0, dequantFuncQ1_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ4_0, dequantFuncQ4_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ4_1, dequantFuncQ4_1_v); } \ - else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ5_0, dequantFuncQ5_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ5_1, dequantFuncQ5_1_v); } \ - else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ8_0, dequantFuncQ8_0_v); } \ - else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ2_K, dequantFuncQ2_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ3_K, dequantFuncQ3_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ4_K, dequantFuncQ4_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ5_K, dequantFuncQ5_K_v); } \ - else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ6_K, dequantFuncQ6_K_v); } \ - else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ1_S, dequantFuncIQ1_S_v); } \ - else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ1_M, dequantFuncIQ1_M_v); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ2_XXS, dequantFuncIQ2_XXS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ2_XS, dequantFuncIQ2_XS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ2_S, dequantFuncIQ2_S_v); } \ - else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ3_XXS, dequantFuncIQ3_XXS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ3_S, dequantFuncIQ3_S_v); } \ - else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ4_XS, dequantFuncIQ4_XS_v); } \ - else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ4_NL, dequantFuncIQ4_NL_v); } \ - else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncMXFP4, dequantFuncMXFP4_v); } \ - else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncNVFP4, dequantFuncNVFP4_v); } + if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, a_q1_0.data, pos_a, tensor_layout, dequantFuncQ1_0, dequantFuncQ1_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, a_q4_0.data, pos_a, tensor_layout, dequantFuncQ4_0, dequantFuncQ4_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, a_q4_1.data, pos_a, tensor_layout, dequantFuncQ4_1, dequantFuncQ4_1_v); } \ + else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, a_q5_0.data, pos_a, tensor_layout, dequantFuncQ5_0, dequantFuncQ5_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, a_q5_1.data, pos_a, tensor_layout, dequantFuncQ5_1, dequantFuncQ5_1_v); } \ + else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, a_q8_0.data, pos_a, tensor_layout, dequantFuncQ8_0, dequantFuncQ8_0_v); } \ + else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, a_q2_k.data, pos_a, tensor_layout, dequantFuncQ2_K, dequantFuncQ2_K_v); } \ + else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K, dequantFuncQ3_K_v); } \ + else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K, dequantFuncQ4_K_v); } \ + else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K, dequantFuncQ5_K_v); } \ + else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K, dequantFuncQ6_K_v); } \ + else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, a_iq1_s.data, pos_a, tensor_layout, dequantFuncIQ1_S, dequantFuncIQ1_S_v); } \ + else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, a_iq1_m.data, pos_a, tensor_layout, dequantFuncIQ1_M, dequantFuncIQ1_M_v); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, a_iq2_xxs.data, pos_a, tensor_layout, dequantFuncIQ2_XXS, dequantFuncIQ2_XXS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, a_iq2_xs.data, pos_a, tensor_layout, dequantFuncIQ2_XS, dequantFuncIQ2_XS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, a_iq2_s.data, pos_a, tensor_layout, dequantFuncIQ2_S, dequantFuncIQ2_S_v); } \ + else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, a_iq3_xxs.data, pos_a, tensor_layout, dequantFuncIQ3_XXS, dequantFuncIQ3_XXS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, a_iq3_s.data, pos_a, tensor_layout, dequantFuncIQ3_S, dequantFuncIQ3_S_v); } \ + else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, a_iq4_xs.data, pos_a, tensor_layout, dequantFuncIQ4_XS, dequantFuncIQ4_XS_v); } \ + else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, a_iq4_nl.data, pos_a, tensor_layout, dequantFuncIQ4_NL, dequantFuncIQ4_NL_v); } \ + else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, a_mxfp4.data, pos_a, tensor_layout, dequantFuncMXFP4, dequantFuncMXFP4_v); } \ + else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, a_nvfp4.data, pos_a, tensor_layout, dequantFuncNVFP4, dequantFuncNVFP4_v); } #else #define COOPMAT_LOAD_A(mat, tensor_layout) \ - if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ1_0); } \ - else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ4_0); } \ - else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ4_1); } \ - else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ5_0); } \ - else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ5_1); } \ - else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ8_0); } \ - else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ2_K); } \ - else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ3_K); } \ - else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ4_K); } \ - else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ5_K); } \ - else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncQ6_K); } \ - else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ1_S); } \ - else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ1_M); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ2_XXS); } \ - else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ2_XS); } \ - else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ2_S); } \ - else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ3_XXS); } \ - else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ3_S); } \ - else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ4_XS); } \ - else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncIQ4_NL); } \ - else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncMXFP4); } \ - else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, data_a, pos_a, tensor_layout, dequantFuncNVFP4); } + if (MmTypeA == GGML_TYPE_Q1_0) { coopMatLoadTensorNV(mat, a_q1_0.data, pos_a, tensor_layout, dequantFuncQ1_0); } \ + else if (MmTypeA == GGML_TYPE_Q4_0) { coopMatLoadTensorNV(mat, a_q4_0.data, pos_a, tensor_layout, dequantFuncQ4_0); } \ + else if (MmTypeA == GGML_TYPE_Q4_1) { coopMatLoadTensorNV(mat, a_q4_1.data, pos_a, tensor_layout, dequantFuncQ4_1); } \ + else if (MmTypeA == GGML_TYPE_Q5_0) { coopMatLoadTensorNV(mat, a_q5_0.data, pos_a, tensor_layout, dequantFuncQ5_0); } \ + else if (MmTypeA == GGML_TYPE_Q5_1) { coopMatLoadTensorNV(mat, a_q5_1.data, pos_a, tensor_layout, dequantFuncQ5_1); } \ + else if (MmTypeA == GGML_TYPE_Q8_0) { coopMatLoadTensorNV(mat, a_q8_0.data, pos_a, tensor_layout, dequantFuncQ8_0); } \ + else if (MmTypeA == GGML_TYPE_Q2_K) { coopMatLoadTensorNV(mat, a_q2_k.data, pos_a, tensor_layout, dequantFuncQ2_K); } \ + else if (MmTypeA == GGML_TYPE_Q3_K) { coopMatLoadTensorNV(mat, a_q3_k.data, pos_a, tensor_layout, dequantFuncQ3_K); } \ + else if (MmTypeA == GGML_TYPE_Q4_K) { coopMatLoadTensorNV(mat, a_q4_k.data, pos_a, tensor_layout, dequantFuncQ4_K); } \ + else if (MmTypeA == GGML_TYPE_Q5_K) { coopMatLoadTensorNV(mat, a_q5_k.data, pos_a, tensor_layout, dequantFuncQ5_K); } \ + else if (MmTypeA == GGML_TYPE_Q6_K) { coopMatLoadTensorNV(mat, a_q6_k.data, pos_a, tensor_layout, dequantFuncQ6_K); } \ + else if (MmTypeA == GGML_TYPE_IQ1_S) { coopMatLoadTensorNV(mat, a_iq1_s.data, pos_a, tensor_layout, dequantFuncIQ1_S); } \ + else if (MmTypeA == GGML_TYPE_IQ1_M) { coopMatLoadTensorNV(mat, a_iq1_m.data, pos_a, tensor_layout, dequantFuncIQ1_M); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XXS) { coopMatLoadTensorNV(mat, a_iq2_xxs.data, pos_a, tensor_layout, dequantFuncIQ2_XXS); } \ + else if (MmTypeA == GGML_TYPE_IQ2_XS) { coopMatLoadTensorNV(mat, a_iq2_xs.data, pos_a, tensor_layout, dequantFuncIQ2_XS); } \ + else if (MmTypeA == GGML_TYPE_IQ2_S) { coopMatLoadTensorNV(mat, a_iq2_s.data, pos_a, tensor_layout, dequantFuncIQ2_S); } \ + else if (MmTypeA == GGML_TYPE_IQ3_XXS) { coopMatLoadTensorNV(mat, a_iq3_xxs.data, pos_a, tensor_layout, dequantFuncIQ3_XXS); } \ + else if (MmTypeA == GGML_TYPE_IQ3_S) { coopMatLoadTensorNV(mat, a_iq3_s.data, pos_a, tensor_layout, dequantFuncIQ3_S); } \ + else if (MmTypeA == GGML_TYPE_IQ4_XS) { coopMatLoadTensorNV(mat, a_iq4_xs.data, pos_a, tensor_layout, dequantFuncIQ4_XS); } \ + else if (MmTypeA == GGML_TYPE_IQ4_NL) { coopMatLoadTensorNV(mat, a_iq4_nl.data, pos_a, tensor_layout, dequantFuncIQ4_NL); } \ + else if (MmTypeA == GGML_TYPE_MXFP4) { coopMatLoadTensorNV(mat, a_mxfp4.data, pos_a, tensor_layout, dequantFuncMXFP4); } \ + else if (MmTypeA == GGML_TYPE_NVFP4) { coopMatLoadTensorNV(mat, a_nvfp4.data, pos_a, tensor_layout, dequantFuncNVFP4); } #endif #endif #else