furyhawk/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-21 10:15:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

opencl: Support broadcast for Adreno MUL_MAT and honor view_offs for Adreno Q8_0 MUL_MAT for llama-server multi-stream (#25910)

Browse Source 
* opencl: handle broadcast for adreno gemm/gemv_noshuffle

* opencl: honor view_offs for adreno noshuffle gemm/gemv

* opencl: general GEMM/GEMV support broadcast

* opencl: remove unnecessary tests

* opencl: remove unnecessary comments

---------

Co-authored-by: Li He <lih@qti.qualcomm.com>
This commit is contained in:
Hongqiang Wang
2026-07-19 22:48:57 -07:00
committed by GitHub
co-authored by Li He
parent 571d0d540d
commit 178a6c4493
1 changed files with 22 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-opencl/ggml-opencl.cpp
+22 -5
View File
@@ -17085,9 +17085,6 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
cl_ulong offset1 = extra1->offset + src1->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
GGML_ASSERT(src1->view_offs == 0);
GGML_ASSERT(dst->view_offs == 0);
const int ne00 = src0->ne[0];
const int ne01 = src0->ne[1];
const int ne02 = src0->ne[2];
@@ -17148,9 +17145,9 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));
@@ -18545,6 +18542,26 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
GGML_ASSERT(ne00 == ne10);
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
// adreno GEMM/GEMV kernels do not support broadcast, assuming ne2 and ne3 are 1 for src1
// so we handle broadcast here
if ((ne12 > 1 || ne13 > 1) && ne02 == 1 && ne03 == 1 &&
src0t != GGML_TYPE_F16 && src0t != GGML_TYPE_F32) {
for (int i13 = 0; i13 < ne13; ++i13) {
for (int i12 = 0; i12 < ne12; ++i12) {
ggml_tensor s1 = *src1;
s1.ne[2] = 1; s1.ne[3] = 1;
s1.view_offs = src1->view_offs + (size_t)i12*nb12 + (size_t)i13*nb13;
ggml_tensor d = *dst;
d.ne[2] = 1; d.ne[3] = 1;
d.view_offs = dst->view_offs + (size_t)i12*nb2 + (size_t)i13*nb3;
ggml_cl_mul_mat(backend, src0, &s1, &d);
}
}
return;
}
#endif
int nth0 = 32;
int nth1 = 1;
int nrows = 1;