From b2dd28a3b601417b8ea345c4db72df3048e1114f Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Thu, 16 Jul 2026 09:28:04 -0700 Subject: [PATCH] hexagon: L2 cache handling rework (dirty bit tracking with lazy flushing) and more MUL_MAT updates (#25762) * hex-mm: fix artificial limit in the solver that restricted number of act-prep threads * hex-mm: fix warning * hex-prof: do not apply --top to the timeline report * hmx-mm: add suport for tiled act-processing to better distribute hvx work * hex-l2: add tracing for l2flush events * workqueue: redo the legacy workpool api to match hmx-queue and dma-queue * hmx-mm: fix f32 activation buffer alignmnet for nhvx=5,6,7 * hex-work: minor cleanup for work-queue apis * hex-work: further cleanup of the work-queue api * hex-l2: optimize l2flushes at the opbatch level * hex-work: remove unused mask * hex-work: no need to drop hvx ctx in the work-queue * hex-work: add explicit wakeup/suspend and make threads spin * hex-bufs: mark any non-weight tensor as compute * hex-dma: dma-queue support for alias queues and cached dma * hex-l2: track tensor aliases and delay or skip flushes as much as possible * hex-l2: simplify tensor alias handling * hex-l2: handle overlapping views as a circular list of aliases * hex-tens: add flags helper * hex-l2: add helper for marking tensors clearn/dirty * hex-l2: mark binary and rope outputs as l2-clean and keep the rest as is for now * hex-l2: proper support for handling all tensor overlap scenarios * hex-trace: instrument matmul init code and cleanup trace checks * hex-thread: introduce dedicated main thread with explicit stack and priority * hex-l2: track dirty state as bitmap and introduce threaded flush * hex-trace: remove redundant checks for ctx != null * hex-l2: allocate entire context as one buffer and l2fetch it after big flushes * hex-l2: disable tensor clearing in binary and rope for now seems to cause issues with fusion * hmx-mm: update act proc to use fastdivs and fix DMA overflow * hmx-mm: make MUL_MAT_ID kernels robust to multi-chunk cases (start_row>0) * hex-queue: remove obsolete queue interfaces and flush hmx-queue at the end of the op-batch * hex-queue: dont use early wakeup for small op-batches * hex-tensors: properly cap max_tensors in op-batches and dirty_map * hex-l2: make sure threaded l2flush does proper rounding * hex-l2: factor out htp_tensor_flush for reuse (if needed) * hex-l2: optimize tensor flushes by coalescing flush-all * hex-l2: optimize multi-threaded flush * hex-drv: futureproof version checks * hexagon: fix errors and warnings on windows * hex-main: update main thread to only use dspqueue_read, dspqueue_peek is not available on some platforms * hex-main: add fallback mode for dspqueue with callbacks * hex-main: introduce fallback mode for using dspqueue callbacks for full op processing * hex-main: remove early wakeup, not helping and seems to cause some errors with certain batch sizes * hex-l2: make sure to use invalidate version of flushall * hex-l2: dont try to trace early l2flush at the start of op-batch * hex-main: remove offset_ctx that must be zero anyway * hex-hmx: fix hmx_queue_depth to use idx_write - idx_read * hex-hmx: use atomic_load for idx_read/write * hex-main: add static assert to make sure n_threads are aligned --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 325 ++++--- ggml/src/ggml-hexagon/htp-drv.cpp | 47 +- ggml/src/ggml-hexagon/htp-drv.h | 8 +- ggml/src/ggml-hexagon/htp/CMakeLists.txt | 9 +- ggml/src/ggml-hexagon/htp/act-ops.c | 11 +- ggml/src/ggml-hexagon/htp/argsort-ops.c | 4 +- ggml/src/ggml-hexagon/htp/binary-ops.c | 1 + ggml/src/ggml-hexagon/htp/cumsum-ops.c | 11 +- ggml/src/ggml-hexagon/htp/dma-queue.c | 106 +++ ggml/src/ggml-hexagon/htp/dma-queue.h | 396 +++++++++ ggml/src/ggml-hexagon/htp/flash-attn-ops.c | 44 +- ggml/src/ggml-hexagon/htp/hex-bitmap.h | 24 + ggml/src/ggml-hexagon/htp/hex-dma.c | 63 -- ggml/src/ggml-hexagon/htp/hex-dma.h | 377 +------- ggml/src/ggml-hexagon/htp/hex-profile.h | 10 +- ggml/src/ggml-hexagon/htp/hex-utils.h | 31 +- .../ggml-hexagon/htp/hmx-mm-kernels-tiled.h | 81 +- ggml/src/ggml-hexagon/htp/hmx-queue.c | 57 +- ggml/src/ggml-hexagon/htp/hmx-queue.h | 41 +- ggml/src/ggml-hexagon/htp/htp-ctx.h | 23 +- ggml/src/ggml-hexagon/htp/htp-ops.h | 11 +- ggml/src/ggml-hexagon/htp/htp-tensor.c | 204 +++++ ggml/src/ggml-hexagon/htp/htp-tensor.h | 36 + ggml/src/ggml-hexagon/htp/main.c | 807 +++++++++++------- ggml/src/ggml-hexagon/htp/matmul-ops.c | 467 ++++++++-- ggml/src/ggml-hexagon/htp/matmul-ops.h | 132 +++ ggml/src/ggml-hexagon/htp/rope-ops.c | 11 +- ggml/src/ggml-hexagon/htp/unary-ops.c | 15 +- ggml/src/ggml-hexagon/htp/work-queue.c | 244 ++++++ ggml/src/ggml-hexagon/htp/work-queue.h | 38 + ggml/src/ggml-hexagon/htp/worker-pool.c | 305 ------- ggml/src/ggml-hexagon/htp/worker-pool.h | 65 -- scripts/snapdragon/ggml-hexagon-profile.py | 4 - 33 files changed, 2476 insertions(+), 1532 deletions(-) create mode 100644 ggml/src/ggml-hexagon/htp/dma-queue.c create mode 100644 ggml/src/ggml-hexagon/htp/dma-queue.h create mode 100644 ggml/src/ggml-hexagon/htp/hex-bitmap.h delete mode 100644 ggml/src/ggml-hexagon/htp/hex-dma.c create mode 100644 ggml/src/ggml-hexagon/htp/htp-tensor.c create mode 100644 ggml/src/ggml-hexagon/htp/htp-tensor.h create mode 100644 ggml/src/ggml-hexagon/htp/work-queue.c create mode 100644 ggml/src/ggml-hexagon/htp/work-queue.h delete mode 100644 ggml/src/ggml-hexagon/htp/worker-pool.c delete mode 100644 ggml/src/ggml-hexagon/htp/worker-pool.h diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 76c71d7ee7..9388a59c1a 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -21,6 +21,11 @@ #include #ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include # include #else # include @@ -28,7 +33,9 @@ #endif #pragma clang diagnostic ignored "-Wnested-anon-types" +#pragma clang diagnostic ignored "-Wlanguage-extension-token" #pragma clang diagnostic ignored "-Wgnu-anonymous-struct" +#pragma clang diagnostic ignored "-Wmicrosoft-enum-value" #include #include @@ -134,6 +141,8 @@ static const char * htp_event_name(uint16_t id) { case HTP_TRACE_EVT_HVX_FA_K_PREP: return "HVX_K_PREP"; case HTP_TRACE_EVT_HVX_FA_V_PREP: return "HVX_V_PREP"; case HTP_TRACE_EVT_HMX_COMP: return "HMX_COMP"; + case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH"; + case HTP_TRACE_EVT_INIT: return "INIT"; default: return "UNKNOWN"; } } @@ -501,6 +510,8 @@ static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t size) { } } } + + GGML_UNUSED(size); } // repack q4_0_tiled tensor into q4_0 data @@ -554,6 +565,8 @@ static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t size) { } } } + + GGML_UNUSED(size); } // repack q4_1 data into q4_1_tiled tensor @@ -611,6 +624,8 @@ static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t size) { } } } + + GGML_UNUSED(size); } // repack q4_1_tiled tensor into q4_1 data @@ -665,6 +680,8 @@ static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t size) { } } } + + GGML_UNUSED(size); } // repack q8_0 data into q8_0_tiled tensor @@ -711,6 +728,8 @@ static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t size) { } } } + + GGML_UNUSED(size); } // repack q8_0_tiled tensor into q8_0 data @@ -761,6 +780,8 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t size) { } } } + + GGML_UNUSED(size); } // repack mxfp4 data into mxfp4_tiled tensor @@ -812,6 +833,8 @@ static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t size) } } } + + GGML_UNUSED(size); } // repack mxfp4_tiled tensor into mxfp4 data @@ -865,6 +888,8 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t size) } } } + + GGML_UNUSED(size); } static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, @@ -965,11 +990,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { + // we might optimize this later, for now take the slow path (ie get/set_tensor) + return false; + GGML_UNUSED(buffer); GGML_UNUSED(src); GGML_UNUSED(dst); - // we might optimize this later, for now take the slow path (ie get/set_tensor) - return false; } static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { @@ -1025,9 +1051,9 @@ static ggml_backend_buffer_t ggml_backend_hexagon_repack_buffer_type_alloc_buffe } } -static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { +static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { return 128; // HVX alignment - GGML_UNUSED(buffer_type); + GGML_UNUSED(buft); } static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { @@ -1039,20 +1065,24 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffe return ggml_row_size(t->type, ne0) * ne1 * ne2 * ne3; } return ggml_nbytes(t); + + GGML_UNUSED(buft); } -static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) { - auto * context = static_cast(buffer_type->context); +static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + auto * context = static_cast(buft->context); return context->sess->max_bufsize; } static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { return opt_hostbuf; + GGML_UNUSED(buft); } static bool ggml_backend_hexagon_repack_buffer_type_is_host(ggml_backend_buffer_type_t buft) { return false; + GGML_UNUSED(buft); } @@ -1098,6 +1128,14 @@ struct ggml_hexagon_opbatch { std::unordered_map t_map; // tensor ptr to index std::unordered_multimap d_map; // tensor data to index + struct tensor_range { + uint64_t start; + uint64_t end; + int bi; + std::vector tensors; + }; + std::vector ranges; + unsigned int n_bufs; // num buffers in the batch unsigned int n_tens; // num tensors ... unsigned int n_ops; // num ops ... @@ -1117,6 +1155,7 @@ struct ggml_hexagon_opbatch { b_map.clear(); t_map.clear(); d_map.clear(); + ranges.clear(); } ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) { @@ -1124,7 +1163,7 @@ struct ggml_hexagon_opbatch { n_bufs_max = HTP_OP_MAX_BUFS; n_ops_max = batch_size; - n_tens_max = n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS; + n_tens_max = std::min(n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS, HTP_OP_MAX_TENSORS); b_vmem_max = max_vmem; @@ -1170,6 +1209,71 @@ struct ggml_hexagon_opbatch { return bi; } + void add_range(const htp_tensor * h, int ti) { + uint64_t t_start = h->data; + uint64_t t_end = t_start + h->size; + int bi = h->bi; + + int first_match = -1; + int unused_idx = -1; + for (size_t i = 0; i < ranges.size(); i++) { + if (ranges[i].bi == -1) { + unused_idx = i; + continue; + } + if (ranges[i].bi != bi) { + continue; + } + if (ranges[i].start >= t_end || ranges[i].end <= t_start) { + continue; + } + + if (first_match == -1) { + first_match = i; + HEX_VERBOSE("ggml-hex: %s range-grow #%d : bi %d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n", + sess->c_name(), (int) i, ranges[i].bi, + (void *) (h_bufs[ranges[i].bi].base + ranges[i].start), + (void *) (h_bufs[ranges[i].bi].base + ranges[i].end), + ti, + (void *) (h_bufs[bi].base + t_start), + (void *) (h_bufs[bi].base + t_end), + (void *) (h_bufs[ranges[i].bi].base + std::min(ranges[i].start, t_start)), + (void *) (h_bufs[ranges[i].bi].base + std::max(ranges[i].end, t_end))); + + ranges[i].start = std::min(ranges[i].start, t_start); + ranges[i].end = std::max(ranges[i].end, t_end); + ranges[i].tensors.push_back(ti); + } else { + HEX_VERBOSE("ggml-hex: %s range-merge #%d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n", + sess->c_name(), first_match, + (void *) (h_bufs[bi].base + ranges[first_match].start), + (void *) (h_bufs[bi].base + ranges[first_match].end), + (int) i, + (void *) (h_bufs[bi].base + ranges[i].start), + (void *) (h_bufs[bi].base + ranges[i].end), + (void *) (h_bufs[bi].base + std::min(ranges[first_match].start, ranges[i].start)), + (void *) (h_bufs[bi].base + std::max(ranges[first_match].end, ranges[i].end))); + + ranges[first_match].start = std::min(ranges[first_match].start, ranges[i].start); + ranges[first_match].end = std::max(ranges[first_match].end, ranges[i].end); + ranges[first_match].tensors.insert( + ranges[first_match].tensors.end(), + ranges[i].tensors.begin(), + ranges[i].tensors.end() + ); + ranges[i].bi = -1; + } + } + + if (first_match == -1) { + if (unused_idx != -1) { + ranges[unused_idx] = {t_start, t_end, bi, {ti}}; + } else { + ranges.push_back({t_start, t_end, bi, {ti}}); + } + } + } + bool same_shape(const htp_tensor * h, const ggml_tensor * t) const { int64_t ne0 = t->ne[0]; int64_t ne1 = t->ne[1]; @@ -1213,6 +1317,7 @@ struct ggml_hexagon_opbatch { htp_tensor &h = h_tens[ti]; h.bi = add_buffer(sbuf); + h.ti = ti; h.data = t_offset; h.type = t->type; @@ -1235,8 +1340,11 @@ struct ggml_hexagon_opbatch { h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3]; } + h.alias = ti; + add_range(&h, ti); + h.flags = 0; - if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + if (ggml_backend_buffer_get_usage(t->buffer) != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { h.flags |= HTP_TENSOR_COMPUTE; } @@ -1313,6 +1421,17 @@ struct ggml_hexagon_opbatch { o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff; } } + + void finalize_ranges() { + for (const auto & r : ranges) { + if (r.bi == -1) { + continue; + } + for (size_t i = 0; i < r.tensors.size(); i++) { + h_tens[r.tensors[i]].alias = r.tensors[(i + 1) % r.tensors.size()]; + } + } + } }; struct ggml_hexagon_opqueue { @@ -1571,6 +1690,8 @@ void ggml_hexagon_session::flush_pending(bool all) { void ggml_hexagon_session::flush_batch() { if (op_batch->empty()) { return; } + op_batch->finalize_ranges(); + htp_opbatch_req req {}; dspqueue_buffer dbuf{}; @@ -1647,7 +1768,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str()); - domain * my_domain = get_domain(this->domain_id); + domain * my_domain = htpdrv_get_domain(this->domain_id); if (my_domain == NULL) { GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP\n"); throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)"); @@ -1793,16 +1914,6 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { } } - if (opt_profile) { - htp_iface_pmu_conf pmu_conf{}; - std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events); - - err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err); - } - } - // Allocate buffers and state for op batching this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); @@ -1821,6 +1932,16 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); } this->valid_iface = true; + + if (opt_profile) { + htp_iface_pmu_conf pmu_conf{}; + std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events); + + err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err); + } + } } void ggml_hexagon_session::release() noexcept(true) { @@ -1929,6 +2050,8 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible( } return true; + + GGML_UNUSED(sinks); } static bool ggml_hexagon_precompute_flash_attn_params( @@ -2149,8 +2272,9 @@ static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_ses return false; } - GGML_UNUSED(sess); return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_matmul_is_hmx_eligible( @@ -2198,6 +2322,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible( } return true; + + GGML_UNUSED(dst); } static bool ggml_hexagon_precompute_hmx_mm_params( @@ -2234,109 +2360,15 @@ static bool ggml_hexagon_precompute_hmx_mm_params( if (is_batched_val && wtype == GGML_TYPE_F16 && group_size > 1) { // Try grouped path first const bool use_dma_activation = (src1->nb[1]/sizeof(float) > (size_t)ne00_padded); - size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; - size_t best_m_chunk = 0; - size_t best_n_chunk = 0; - size_t best_vtcm_size = 0; - - int act_threads = n_threads; - while (act_threads >= 1) { - const size_t f32_scratch_size = use_dma_activation ? hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE) : 0; - size_t group_overhead = 256 + f32_scratch_size; - size_t group_size_per_n, group_size_per_m, group_size_per_mn; - htp_mm_hmx_get_batched_chunk_costs(ne00_padded, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); - - size_t m_chunk_candidate = 0; - size_t n_chunk_candidate = 0; - size_t vtcm_size_candidate = 0; - - if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, - (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, - &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads); - if (exact_size <= vtcm_budget) { - size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { - best_mblocks = mblocks; - best_act_threads = act_threads; - best_m_chunk = m_chunk_candidate; - best_n_chunk = n_chunk_candidate; - best_vtcm_size = exact_size; - } - } - } - if (act_threads == 1) { - act_threads = 0; - } else { - act_threads /= 2; - } - } - - if (best_act_threads > 0) { - m_chunk = best_m_chunk; - n_chunk = best_n_chunk; - vtcm_size = best_vtcm_size; - act_threads_selected = best_act_threads; + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, group_size, use_dma_activation, n_threads, pipeline, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { use_grouped = true; } } if (!use_grouped) { // Fallback to simple 2D path (group_size = 1) - size_t best_mblocks = SIZE_MAX; - int best_act_threads = 0; - size_t best_m_chunk = 0; - size_t best_n_chunk = 0; - size_t best_vtcm_size = 0; - - // For MUL_MAT_ID the kernel runs one 2D matmul per expert, with M equal to the number of rows routed to that expert. - // A single expert can receive up to all routed rows (dst->ne[1]*dst->ne[2] = n_expert_used*n_tokens), so size the chunk - // search for that upper bound rather than ne12 (token positions only). - // We recompute m_chunk per expert against the actual count in the NPU kernel. - const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]); - const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; - const int m_for_cost = is_matmul_id ? m_id_rows : ne11; - - int act_threads = n_threads; - while (act_threads >= 1) { - const size_t act_f32_size = is_matmul_id ? 0 : hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE); - size_t simple_2d_overhead = 256 + act_f32_size; - size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; - htp_mm_hmx_get_2d_chunk_costs(wtype, ne00_padded, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); - - size_t m_chunk_candidate = 0; - size_t n_chunk_candidate = 0; - size_t vtcm_size_candidate = 0; - - if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, - (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, - &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { - size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size); - if (exact_size <= vtcm_budget) { - size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; - if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { - best_mblocks = mblocks; - best_act_threads = act_threads; - best_m_chunk = m_chunk_candidate; - best_n_chunk = n_chunk_candidate; - best_vtcm_size = exact_size; - } - } - } - if (act_threads == 1) { - act_threads = 0; - } else { - act_threads /= 2; - } - } - - if (best_act_threads > 0) { - m_chunk = best_m_chunk; - n_chunk = best_n_chunk; - vtcm_size = best_vtcm_size; - act_threads_selected = best_act_threads; - } else { + const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]); + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, ne11_padded, ne11, n_threads, pipeline, is_matmul_id, aligned_tile_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { return false; } } @@ -2352,6 +2384,8 @@ static bool ggml_hexagon_precompute_hmx_mm_params( kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); kparams->vtcm_size = vtcm_size; kparams->vtcm_src0_size = 0; + kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected); + kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); kparams->vtcm_src1_size = 0; kparams->vtcm_dst_size = 0; @@ -2361,6 +2395,8 @@ static bool ggml_hexagon_precompute_hmx_mm_params( kparams->kernel_type = HTP_MM_KERNEL_HMX_2D; } return true; + + GGML_UNUSED(src0); } static void ggml_hexagon_precompute_hvx_mm_params( @@ -2955,6 +2991,8 @@ static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * se } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -2981,6 +3019,8 @@ static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * se } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3006,6 +3046,8 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3025,10 +3067,11 @@ static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * } return true; + + GGML_UNUSED(sess); } -static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op) { +static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * src1 = op->src[1]; const struct ggml_tensor * dst = op; @@ -3057,6 +3100,8 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3122,6 +3167,8 @@ static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * s } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3142,6 +3189,8 @@ static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3162,6 +3211,8 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3182,6 +3233,8 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3243,6 +3296,8 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess return false; } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3282,6 +3337,8 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * } return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3292,8 +3349,9 @@ static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, return false; } - GGML_UNUSED(sess); return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3308,8 +3366,9 @@ static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * se return false; } - GGML_UNUSED(sess); return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3331,8 +3390,9 @@ static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess return false; } - GGML_UNUSED(sess); return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3364,8 +3424,9 @@ static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * return false; } - GGML_UNUSED(sess); return true; + + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -3812,6 +3873,8 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr } } } + + GGML_UNUSED(backend); } static struct ggml_backend_i hexagon_backend_i = { @@ -3930,6 +3993,8 @@ static bool ggml_hexagon_supported_buffers(ggml_hexagon_session *sess, const str } static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { + GGML_UNUSED(sess); + const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * dst = op; @@ -4000,6 +4065,7 @@ static bool ggml_hexagon_supported_concat(const struct ggml_hexagon_session * se } return true; + GGML_UNUSED(sess); } static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -4009,8 +4075,8 @@ static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess return false; } - GGML_UNUSED(sess); return true; + GGML_UNUSED(sess); } static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { @@ -4293,6 +4359,7 @@ static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, cons } return NULL; + GGML_UNUSED(reg); } template std::vector str_to_vec(const char* str) { @@ -4351,10 +4418,18 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { // Init Arch first since it affects other defaults if (!str_arch) { - int err = get_hex_arch_ver(CDSP_DOMAIN_ID, &opt_arch); + int err = htpdrv_get_arch(CDSP_DOMAIN_ID, &opt_arch); if (err != 0) { GGML_LOG_ERROR("ggml-hex: failed to query HTP version (err %d) defaulting to v73\n", err); opt_arch = 73; + } else { + if (opt_arch < 73) { + GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is under supported range, capping at v73\n", opt_arch); + opt_arch = 73; + } else if (opt_arch > 81) { + GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v81\n", opt_arch); + opt_arch = 81; + } } } else { if (str_arch[0] == 'v' || str_arch[0] == 'V') { diff --git a/ggml/src/ggml-hexagon/htp-drv.cpp b/ggml/src/ggml-hexagon/htp-drv.cpp index 4c376b5fc9..40c6010540 100644 --- a/ggml/src/ggml-hexagon/htp-drv.cpp +++ b/ggml/src/ggml-hexagon/htp-drv.cpp @@ -1,13 +1,8 @@ -// sample drv interface - -#pragma clang diagnostic ignored "-Wgnu-anonymous-struct" -#pragma clang diagnostic ignored "-Wmissing-prototypes" -#pragma clang diagnostic ignored "-Wsign-compare" - #include #include #include #include + #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN # ifndef NOMINMAX @@ -16,9 +11,17 @@ # include # include #else -# include -# include +# include +# include #endif + +#pragma clang diagnostic ignored "-Wgnu-anonymous-struct" +#pragma clang diagnostic ignored "-Wmissing-prototypes" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wlanguage-extension-token" +#pragma clang diagnostic ignored "-Wmicrosoft-enum-value" +#pragma clang diagnostic ignored "-Wnested-anon-types" + #include "ggml-impl.h" #include "htp-drv.h" #include "libdl.h" @@ -359,7 +362,7 @@ int htpdrv_init() { return AEE_SUCCESS; } -domain * get_domain(int domain_id) { +domain * htpdrv_get_domain(int domain_id) { int i = 0; int size = sizeof(supported_domains) / sizeof(domain); @@ -372,7 +375,7 @@ domain * get_domain(int domain_id) { return NULL; } -int get_hex_arch_ver(int domain, int * arch) { +int htpdrv_get_arch(int domain, int * arch) { if (!remote_handle_control_pfn) { GGML_LOG_ERROR("ggml-hex: remote_handle_control is not supported on this device\n"); return AEE_EUNSUPPORTEDAPI; @@ -394,25 +397,7 @@ int get_hex_arch_ver(int domain, int * arch) { return err; } - switch (arch_ver.capability & 0xff) { - case 0x68: - *arch = 68; - return 0; - case 0x69: - *arch = 69; - return 0; - case 0x73: - *arch = 73; - return 0; - case 0x75: - *arch = 75; - return 0; - case 0x79: - *arch = 79; - return 0; - case 0x81: - *arch = 81; - return 0; - } - return -1; + uint32_t val = arch_ver.capability & 0xff; + *arch = (int) ((val >> 4) * 10 + (val & 0x0f)); + return 0; } diff --git a/ggml/src/ggml-hexagon/htp-drv.h b/ggml/src/ggml-hexagon/htp-drv.h index 6eba7ba17d..f3cc0da75c 100644 --- a/ggml/src/ggml-hexagon/htp-drv.h +++ b/ggml/src/ggml-hexagon/htp-drv.h @@ -96,17 +96,17 @@ extern "C" { HTPDRV_API int htpdrv_init(void); /** - * get_domain API: get domain struct from domain value. + * htpdrv_get_domain API: get domain struct from domain value. * * @param[in] domain value of a domain * @return Returns domain struct of the domain if it is supported or else * returns NULL. * */ -HTPDRV_API domain * get_domain(int domain_id); +HTPDRV_API domain * htpdrv_get_domain(int domain_id); /** - * get_hex_arch_ver API: query the Hexagon processor architecture version information + * htpdrv_get_arch API: query the Hexagon processor architecture version information * * @param[in] domain_id value of a domain * @param[out] Arch version (73, 75, ...) @@ -114,7 +114,7 @@ HTPDRV_API domain * get_domain(int domain_id); * non-zero if error, return value points to the error. * */ -HTPDRV_API int get_hex_arch_ver(int domain, int * arch); +HTPDRV_API int htpdrv_get_arch(int domain, int * arch); #ifdef __cplusplus } diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index cf9e726c00..4fb526f0c0 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -17,9 +17,12 @@ set(HTP_LIB ggml-htp-${DSP_VERSION}) add_library(${HTP_LIB} SHARED main.c htp_iface_skel.c - worker-pool.c - hex-dma.c + work-queue.c + dma-queue.c hmx-queue.c + htp-tensor.c + matmul-ops.c + flash-attn-ops.c gated-delta-net-ops.c binary-ops.c unary-ops.c @@ -38,8 +41,6 @@ add_library(${HTP_LIB} SHARED diag-ops.c solve-tri-ops.c pad-ops.c - flash-attn-ops.c - matmul-ops.c argsort-ops.c ) diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index 6416d2dfbc..e56f2b6a52 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -16,6 +16,7 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "htp-ops.h" +#include "htp-tensor.h" #define htp_act_preamble \ const struct htp_tensor * src0 = actx->octx->src[0]; \ @@ -766,17 +767,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { } int op_activations(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - switch (octx->src[0]->type) { case HTP_TYPE_F32: - err = execute_op_activations_f32(octx); - break; + return execute_op_activations_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 13f12a39e6..774faef5f3 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -345,7 +345,7 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int i int32_t * indices_buf = (int32_t *) (spad + values_size); \ uint32_t nb01 = src0->nb[1]; \ uint32_t nb1 = dst->nb[1]; \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[i]; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); \ for (uint32_t r = start_row; r < end_row; r++) { \ uint32_t src_offset = r * nb01; \ @@ -411,7 +411,7 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data const HVX_Vector ind_init_vec = *(HVX_Vector *)argosrt_ramp_lut; const HVX_Vector ind_diff_vec = Q6_V_vsplat_R(32); - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); for (uint32_t r = start_row; r < end_row; r++) { diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index 52013ad0fe..db61779635 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -16,6 +16,7 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "htp-ops.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index 2ced197123..2d45c39f23 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -9,6 +9,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-types.h" #include "hvx-utils.h" #include "hex-dma.h" @@ -255,16 +256,10 @@ int op_cumsum_f32(struct htp_ops_context * octx) { int op_cumsum(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; - int err = HTP_STATUS_OK; - switch (dst->type) { case HTP_TYPE_F32: - err = op_cumsum_f32(octx); - break; + return op_cumsum_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.c b/ggml/src/ggml-hexagon/htp/dma-queue.c new file mode 100644 index 0000000000..8f60a2c9dc --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dma-queue.c @@ -0,0 +1,106 @@ +#include "dma-queue.h" + +#include +#include +#include + +#pragma clang diagnostic ignored "-Wunused-function" + +static inline uint32_t pow2_ceil(uint32_t x) { + if (x <= 1) { + return 1; + } + int p = 2; + x--; + while (x >>= 1) { + p <<= 1; + } + return p; +} + +static inline uintptr_t align_up(uintptr_t addr, size_t align) { + return (addr + align - 1) & ~(align - 1); +} + +size_t dma_queue_sizeof(size_t capacity) { + capacity = pow2_ceil(capacity); + + size_t size_q = sizeof(dma_queue); + size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r = sizeof(dma_ring); + size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE); + size_t size_desc = capacity * sizeof(dma_descriptor_2d); + size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + size_t size_dptr = capacity * sizeof(dma_ptr); + + return offset_dptr + size_dptr; +} + +size_t dma_queue_alignof(void) { + return HEX_L2_LINE_SIZE; +} + +dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace) { + capacity = pow2_ceil(capacity); + + size_t size_q = sizeof(dma_queue); + size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r = sizeof(dma_ring); + size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE); + size_t size_desc = capacity * sizeof(dma_descriptor_2d); + size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + size_t size_dptr = capacity * sizeof(dma_ptr); + + size_t total_size = offset_dptr + size_dptr; + memset(ptr, 0, total_size); + + dma_queue * q = (dma_queue *) ptr; + dma_ring * r = (dma_ring *) ((uintptr_t) ptr + offset_r); + + q->ring = r; + q->nocache = 0; + q->alias = false; + + r->trace = trace; + r->vtcm_base = vtcm_base; + r->vtcm_end = vtcm_base + vtcm_size; + r->capacity = capacity; + r->idx_mask = capacity - 1; + r->push_idx = 0; + r->pop_idx = 0; + + r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc); + r->dptr = (dma_ptr *) ((uintptr_t) ptr + offset_dptr); + r->tail = &r->desc[capacity - 1]; + + FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", capacity, total_size); + + return q; +} + +void dma_queue_free(dma_queue_t q) { + (void) q; +} + +size_t dma_queue_alias_sizeof(void) { + return sizeof(dma_queue); +} + +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache) { + dma_queue * q = (dma_queue *) ptr; + memset(q, 0, sizeof(dma_queue)); + + q->ring = main_q->ring; + q->nocache = nocache; + q->alias = true; + + return q; +} + +void dma_queue_alias_free(dma_queue_t q) { + (void) q; +} + +void dma_queue_flush(dma_queue_t q) { + while (dma_queue_pop(q).dst != NULL) ; +} diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h new file mode 100644 index 0000000000..dba452cfbf --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -0,0 +1,396 @@ +#ifndef HTP_DMA_H +#define HTP_DMA_H + +#include +#include +#include +#include +#include "hex-utils.h" + +#include "hex-profile.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date +typedef struct dma_descriptor_1d_s { + void * next; + uint32_t size:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + void * src; + void * dst; +} dma_descriptor_1d; + +#if __HVX_ARCH__ < 75 + +typedef struct dma_descriptor_2d_s { + void * next; + uint32_t reserved0:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + void * src; + void * dst; + uint32_t desc_type:8; + uint32_t reserved1:24; + uint32_t row_size:16; + uint32_t nrows:16; + uint32_t src_stride:16; + uint32_t dst_stride:16; + uint32_t src_offset:16; + uint32_t dst_offset:16; +} dma_descriptor_2d; + +#else + +typedef struct dma_descriptor_2d_s { + void * next; + uint32_t dst_stride:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + void * src; + void * dst; + uint32_t desc_type:8; + uint32_t reserved0:24; + uint32_t row_size:24; + uint32_t nrows_lo:8; + uint32_t nrows_hi:8; + uint32_t src_stride:24; + uint32_t offset:24; + uint32_t reserved1:8; +} dma_descriptor_2d; + +#endif + +typedef struct { + void *dst; + const void *src; +} dma_ptr; + +typedef struct dma_ring_s dma_ring; +struct dma_ring_s { + dma_descriptor_2d * desc; // descriptor pointers + dma_descriptor_2d * tail; // tail pointer + dma_ptr * dptr; // dst/src pointers + uint32_t push_idx; + uint32_t pop_idx; + uint32_t capacity; + uint32_t idx_mask; + struct htp_thread_trace * trace; + uintptr_t vtcm_base; + uintptr_t vtcm_end; +}; + +typedef struct dma_queue_s dma_queue; +typedef dma_queue * dma_queue_t; + +struct dma_queue_s { + dma_ring * ring; // Points to the descriptor ring state + uint8_t nocache; // Queue-specific bypass flag + bool alias; // When set, dma_queue_delete will not free the ring +}; + +void dma_queue_flush(dma_queue_t q); + +size_t dma_queue_sizeof(size_t capacity); +size_t dma_queue_alignof(void); +dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace); +void dma_queue_free(dma_queue_t q); + +size_t dma_queue_alias_sizeof(void); +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache); +void dma_queue_alias_free(dma_queue_t q); + +// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead +// but those do not seem to always compiler properly. +static inline void dmstart(void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmstart(%0)" : : "r"(next)); +} + +static inline void dmlink(void * cur, void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next)); +} + +static inline unsigned int dmpoll(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory"); + return ret; +} + +static inline unsigned int dmwait(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory"); + return ret; +} + +static inline dma_ptr dma_make_ptr(void *dst, const void *src) +{ + dma_ptr p = { dst, src }; + return p; +} + +static inline bool dma_is_vtcm(const dma_queue * q, const void * ptr) { + return (uintptr_t) ptr >= q->ring->vtcm_base && (uintptr_t) ptr < q->ring->vtcm_end; +} + +static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) { + dma_ring * r = q->ring; + if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { + FARF(HIGH, "dma-push: queue full\n"); + return false; + } + + dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx]; + desc->src = (void *) dptr.src; + desc->dst = (void *) dptr.dst; + desc->size = size; + + r->dptr[r->push_idx] = dptr; + + if (size) { + desc->next = NULL; + desc->desc_size = 0; // 1D mode + desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache; + desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; + desc->order = 0; + desc->done = 0; + + htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); + dmlink(r->tail, desc); + r->tail = (dma_descriptor_2d *) desc; + } else { + desc->desc_size = 0; + desc->done = 1; + } + + r->push_idx = (r->push_idx + 1) & r->idx_mask; + return true; +} + +static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + dma_ring * r = q->ring; + if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { + FARF(HIGH, "dma-push: queue full\n"); + return false; + } + + dma_descriptor_2d * desc = &r->desc[r->push_idx]; + + desc->next = NULL; + desc->reserved0 = 0; + desc->reserved1 = 0; + desc->desc_size = 1; // 2d mode + desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache; + desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; + desc->src_comp = 0; + desc->dst_comp = 0; + desc->order = 0; + desc->done = 0; + desc->src_stride = src_stride; + desc->dst_stride = dst_stride; + desc->src = (void *) dptr.src; + desc->dst = (void *) dptr.dst; + desc->row_size = row_size; + +#if __HVX_ARCH__ < 75 + desc->desc_type = 0; // 2d (16-bit) mode + desc->nrows = nrows; + desc->src_offset = 0; + desc->dst_offset = 0; +#else + desc->desc_type = 9; // 2d (24-bit) mode + desc->nrows_lo = (nrows & 0xff); + desc->nrows_hi = (nrows >> 8); + desc->offset = 0; +#endif + + r->dptr[r->push_idx] = dptr; + + if (nrows) { + htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); + dmlink(r->tail, desc); + r->tail = desc; + } else { + desc->done = 1; + } + + r->push_idx = (r->push_idx + 1) & r->idx_mask; + return true; +} + +static inline dma_ptr dma_queue_pop(dma_queue * q) { + dma_ring * r = q->ring; + dma_ptr dptr = { NULL }; + + if (r->push_idx == r->pop_idx) { + return dptr; + } + + dma_descriptor_2d * desc = &r->desc[r->pop_idx]; + + // Wait for desc to complete + if (!desc->done) { + while (!desc->done) { + dmpoll(); + } + } + htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx); + + dptr = r->dptr[r->pop_idx]; + + r->pop_idx = (r->pop_idx + 1) & r->idx_mask; + return dptr; +} + +static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) { + dma_ring * r = q->ring; + dma_ptr dptr = { NULL }; + + if (r->push_idx == r->pop_idx) { + return dptr; + } + + dptr = r->dptr[r->pop_idx]; + + r->pop_idx = (r->pop_idx + 1) & r->idx_mask; + return dptr; +} + +static inline bool dma_queue_empty(dma_queue * q) { + return q->ring->push_idx == q->ring->pop_idx; +} + +static inline uint32_t dma_queue_depth(dma_queue * q) { + return (q->ring->push_idx - q->ring->pop_idx) & q->ring->idx_mask; +} + +static inline uint32_t dma_queue_capacity(dma_queue * q) { + return q->ring->capacity; +} + +#if __HVX_ARCH__ < 75 + +// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535. +// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions. + +#define DMA_MAX_FIELD_VAL 65535u + +static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + // Fast path: everything fits in 16 bits + if (nrows == 0 || __builtin_expect( + row_size <= DMA_MAX_FIELD_VAL && + nrows <= DMA_MAX_FIELD_VAL && + src_stride <= DMA_MAX_FIELD_VAL && + dst_stride <= DMA_MAX_FIELD_VAL, 1)) { + return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); + } + + // Contiguous block + // Use 1d DMA mode which supports sizes up to 24-bits (16MB) + if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { + size_t total = row_size * nrows; + return dma_queue_push_single_1d(q, dptr, total); + } + + // Stride overflow - fall back to row-by-row. + { + const uint8_t *src = (const uint8_t *) dptr.src; + uint8_t *dst = (uint8_t *) dptr.dst; + for (size_t r = 0; r < nrows; ++r) { + dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride); + if (!dma_queue_push_single_1d(q, p, row_size)) + return false; + if (r + 1 < nrows) + dma_queue_pop(q); + } + return true; + } +} + +#else // HVX_ARCH >= 75 + +static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + // On v75 and up we always use 2d 24-bit mode + return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); +} + +#endif + +static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { + return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows); +} + +static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { + return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows); +} + +#define DMA_CACHE_MAX_SIZE 256U + +typedef struct { + uint8_t *base; + uint32_t line_size; + uint32_t capacity; + uint32_t src[DMA_CACHE_MAX_SIZE]; + uint16_t age[DMA_CACHE_MAX_SIZE]; +} dma_cache; + +static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity) +{ + c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; + c->base = base; + c->line_size = line_size; + + for (unsigned i=0; i < c->capacity; i++) { + c->src[i] = 0; + c->age[i] = 0; + } +} + +static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) +{ + uint32_t o_idx = 0; + uint16_t o_age = 0; + uint8_t * dst = 0; + + for (unsigned i=0; i < c->capacity; i++) { + if (c->src[i] == (uint32_t) src) { + c->age[i] = 0; + dst = c->base + (i * c->line_size); nrows = 0; // dummy dma + } else { + c->age[i]++; + if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; } + } + } + if (!dst) { + c->age[o_idx] = 0; + c->src[o_idx] = (uint32_t) src; + dst = c->base + o_idx * c->line_size; // normal nrows dma + return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); + } + + return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* HTP_DMA_H */ diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index 6f2a643e69..a3bd0bf8f8 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -24,7 +24,7 @@ #include "hvx-reduce.h" #include "hvx-flash-attn.h" #include "htp-vtcm.h" -#include "worker-pool.h" +#include "work-queue.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" @@ -204,7 +204,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * if (ir0 >= ir1) return; - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; dma_queue * dma = octx->ctx->dma[ith]; @@ -486,7 +486,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles, (const __fp16 *) args->curr_k, total_rows, factx->DK, args->src_stride, start, end); @@ -494,7 +494,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) } static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { n = factx->n_threads; @@ -502,7 +502,7 @@ static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_row uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t }; if (n > 1) { - worker_pool_run_func(wp, fa_k_interleave_thread, &args, n); + work_queue_run(wp, fa_k_interleave_thread, &args, n); } else { fa_k_interleave_thread(1, 0, &args); } @@ -534,7 +534,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst; - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV, args->src_stride, (uint32_t) args->n_col_tiles, start, end); @@ -548,7 +548,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, void * v_tiles_dst, size_t n_col_tiles, uint32_t kv_start) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { n = factx->n_threads; @@ -556,7 +556,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx, uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t }; if (n > 1) { - worker_pool_run_func(wp, fa_v_interleave_thread, &args, n); + work_queue_run(wp, fa_v_interleave_thread, &args, n); } else { fa_v_interleave_thread(1, 0, &args); } @@ -589,7 +589,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { const size_t start = (size_t) i * rows_per_t; const size_t end = hex_smin(start + rows_per_t, factx->g_br); - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); // Parallel initialization of per-block state @@ -720,7 +720,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { n = factx->n_threads; @@ -739,7 +739,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx, args.q_transposed = q->nb[1] < q->nb[2]; atomic_init(&args.barrier, n); if (n > 1) { - worker_pool_run_func(wp, fa_q_load_thread, &args, n); + work_queue_run(wp, fa_q_load_thread, &args, n); } else { fa_q_load_thread(1, 0, &args); } @@ -772,7 +772,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -820,7 +820,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); const struct htp_tensor * dst = args->dst; @@ -862,7 +862,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx, uint32_t kv_head, uint32_t ib3, size_t n_rows_g) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; uint32_t n = 1; if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { n = factx->n_threads; @@ -871,7 +871,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx, fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t }; worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; if (n > 1) { - worker_pool_run_func(wp, store_fn, &args, n); + work_queue_run(wp, store_fn, &args, n); } else { store_fn(1, 0, &args); } @@ -930,7 +930,7 @@ static inline void fa_softmax_impl( return; } - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride @@ -1290,7 +1290,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, fa_softmax_args_t * sargs, size_t n_row_tiles, size_t n_row_tiles_g_br) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; + work_queue_t wp = factx->octx->ctx->work_queue; const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); worker_callback_t softmax_fn = fa_softmax_thread; @@ -1307,7 +1307,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); sargs->thread_div = init_fastdiv_values(n_use); - worker_pool_run_func(wp, softmax_fn, sargs, n_use); + work_queue_run(wp, softmax_fn, sargs, n_use); } else { softmax_fn(1, 0, sargs); } @@ -1519,8 +1519,8 @@ static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) { // ============================================================================ int hmx_flash_attn_ext(struct htp_ops_context * octx) { - struct htp_thread_trace * tr_hvx = octx->ctx ? &octx->ctx->trace[0] : NULL; - struct htp_thread_trace * tr_hmx = octx->ctx ? &octx->ctx->trace[HTP_MAX_NTHREADS] : NULL; + struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0]; + struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS]; const struct htp_tensor * q = octx->src[0]; const struct htp_tensor * k = octx->src[1]; const struct htp_tensor * v = octx->src[2]; @@ -1735,7 +1735,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) { const size_t k_src_stride = size_k_row_padded / sizeof(__fp16); const size_t v_src_stride = size_v_row_padded / sizeof(__fp16); - struct hmx_queue * hmx_q = ctx->hmx_queue; + hmx_queue_t hmx_q = ctx->hmx_queue; if (factx.pipeline) { // Pipeline path @@ -2084,7 +2084,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { } if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - worker_pool_run_func(octx->ctx->worker_pool, flash_attn_ext_f16_thread, &factx, octx->n_threads); + work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads); } return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/hex-bitmap.h b/ggml/src/ggml-hexagon/htp/hex-bitmap.h new file mode 100644 index 0000000000..140898852a --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hex-bitmap.h @@ -0,0 +1,24 @@ +#ifndef HEX_BITMAP_H +#define HEX_BITMAP_H + +#include +#include +#include + +static inline void bitmap_set(uint32_t * bitmap, uint32_t idx) { + bitmap[idx / 32] |= (1U << (idx % 32)); +} + +static inline void bitmap_clear(uint32_t * bitmap, uint32_t idx) { + bitmap[idx / 32] &= ~(1U << (idx % 32)); +} + +static inline bool bitmap_test(const uint32_t * bitmap, uint32_t idx) { + return (bitmap[idx / 32] & (1U << (idx % 32))) != 0; +} + +static inline void bitmap_reset(uint32_t * bitmap, size_t size_in_bits) { + memset(bitmap, 0, ((size_in_bits + 31) / 32) * sizeof(uint32_t)); +} + +#endif // HEX_BITMAP_H diff --git a/ggml/src/ggml-hexagon/htp/hex-dma.c b/ggml/src/ggml-hexagon/htp/hex-dma.c deleted file mode 100644 index b66e2d2603..0000000000 --- a/ggml/src/ggml-hexagon/htp/hex-dma.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "hex-dma.h" - -#include -#include -#include - -#pragma clang diagnostic ignored "-Wunused-function" - -static inline uint32_t pow2_ceil(uint32_t x) { - if (x <= 1) { - return 1; - } - int p = 2; - x--; - while (x >>= 1) { - p <<= 1; - } - return p; -} - -dma_queue * dma_queue_create(size_t capacity) { - dma_queue * q = (dma_queue *) memalign(32, sizeof(dma_queue)); - if (q == NULL) { - FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__); - return NULL; - } - - capacity = pow2_ceil(capacity); - - memset(q, 0, sizeof(dma_queue)); - q->capacity = capacity; - q->idx_mask = capacity - 1; - - q->desc = (dma_descriptor_2d *) memalign(64, capacity * sizeof(dma_descriptor_2d)); - memset(q->desc, 0, capacity * sizeof(dma_descriptor_2d)); - - q->dptr = (dma_ptr *) memalign(4, capacity * sizeof(dma_ptr)); - memset(q->dptr, 0, capacity * sizeof(dma_ptr)); - - q->tail = &q->desc[capacity - 1]; - - if (!q->desc && !q->dptr) { - FARF(ERROR, "%s: failed to allocate DMA queue items\n", __FUNCTION__); - return NULL; - } - - FARF(HIGH, "dma-queue: capacity %u\n", capacity); - - return q; -} - -void dma_queue_delete(dma_queue * q) { - if (!q) { - return; - } - free(q->desc); - free(q->dptr); - free(q); -} - -void dma_queue_flush(dma_queue * q) { - while (dma_queue_pop(q).dst != NULL) ; -} diff --git a/ggml/src/ggml-hexagon/htp/hex-dma.h b/ggml/src/ggml-hexagon/htp/hex-dma.h index 98fcc9fda6..9e9a5f9502 100644 --- a/ggml/src/ggml-hexagon/htp/hex-dma.h +++ b/ggml/src/ggml-hexagon/htp/hex-dma.h @@ -1,375 +1,2 @@ -#ifndef HTP_DMA_H -#define HTP_DMA_H - -#include -#include -#include -#include -#include "hex-utils.h" - -#include "hex-profile.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date -typedef struct dma_descriptor_1d_s { - void * next; - uint32_t size:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; -} dma_descriptor_1d; - -#if __HVX_ARCH__ < 75 - -typedef struct dma_descriptor_2d_s { - void * next; - uint32_t reserved0:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; - uint32_t desc_type:8; - uint32_t reserved1:24; - uint32_t row_size:16; - uint32_t nrows:16; - uint32_t src_stride:16; - uint32_t dst_stride:16; - uint32_t src_offset:16; - uint32_t dst_offset:16; -} dma_descriptor_2d; - -#else - -typedef struct dma_descriptor_2d_s { - void * next; - uint32_t dst_stride:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; - uint32_t desc_type:8; - uint32_t reserved0:24; - uint32_t row_size:24; - uint32_t nrows_lo:8; - uint32_t nrows_hi:8; - uint32_t src_stride:24; - uint32_t offset:24; - uint32_t reserved1:8; -} dma_descriptor_2d; - -#endif - -typedef struct { - void *dst; - const void *src; -} dma_ptr; - -typedef struct { - dma_descriptor_2d * desc; // descriptor pointers - dma_descriptor_2d * tail; // tail pointer - dma_ptr * dptr; // dst/src pointers - uint32_t push_idx; - uint32_t pop_idx; - uint32_t capacity; - uint32_t idx_mask; - struct htp_thread_trace * trace; -} dma_queue; - -dma_queue * dma_queue_create(size_t capacity); -void dma_queue_delete(dma_queue * q); -void dma_queue_flush(dma_queue * q); - -// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead -// but those do not seem to always compiler properly. -static inline void dmstart(void * next) { - asm volatile(" release(%0):at" : : "r"(next)); - asm volatile(" dmstart(%0)" : : "r"(next)); -} - -static inline void dmlink(void * cur, void * next) { - asm volatile(" release(%0):at" : : "r"(next)); - asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next)); -} - -static inline unsigned int dmpoll(void) { - unsigned int ret = 0; - asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory"); - return ret; -} - -static inline unsigned int dmwait(void) { - unsigned int ret = 0; - asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory"); - return ret; -} - -static inline dma_ptr dma_make_ptr(void *dst, const void *src) -{ - dma_ptr p = { dst, src }; - return p; -} - -static const uint32_t dma_src_l2_bypass_on = 1; -static const uint32_t dma_dst_l2_bypass_on = 1; - -static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) { - if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) { - FARF(HIGH, "dma-push: queue full\n"); - return false; - } - - dma_descriptor_1d * desc = (dma_descriptor_1d *) &q->desc[q->push_idx]; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; - desc->size = size; - - q->dptr[q->push_idx] = dptr; - - if (size) { - desc->next = NULL; - desc->desc_size = 0; // 1D mode - desc->src_bypass = dma_src_l2_bypass_on; - desc->dst_bypass = dma_dst_l2_bypass_on; - desc->order = 0; - desc->done = 0; - - htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx); - dmlink(q->tail, desc); - q->tail = (dma_descriptor_2d *) desc; - } else { - desc->desc_size = 0; - desc->done = 1; - } - - q->push_idx = (q->push_idx + 1) & q->idx_mask; - return true; -} - -static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) { - FARF(HIGH, "dma-push: queue full\n"); - return false; - } - - dma_descriptor_2d * desc = &q->desc[q->push_idx]; - - desc->next = NULL; - desc->reserved0 = 0; - desc->reserved1 = 0; - desc->desc_size = 1; // 2d mode - desc->src_bypass = dma_src_l2_bypass_on; - desc->dst_bypass = dma_dst_l2_bypass_on; - desc->src_comp = 0; - desc->dst_comp = 0; - desc->order = 0; - desc->done = 0; - desc->src_stride = src_stride; - desc->dst_stride = dst_stride; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; - desc->row_size = row_size; - -#if __HVX_ARCH__ < 75 - desc->desc_type = 0; // 2d (16-bit) mode - desc->nrows = nrows; - desc->src_offset = 0; - desc->dst_offset = 0; -#else - desc->desc_type = 9; // 2d (24-bit) mode - desc->nrows_lo = (nrows & 0xff); - desc->nrows_hi = (nrows >> 8); - desc->offset = 0; -#endif - - q->dptr[q->push_idx] = dptr; - - if (nrows) { - htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx); - dmlink(q->tail, desc); - q->tail = desc; - } else { - desc->done = 1; - } - - // FARF(ERROR, "dma-push: i %u row-size %u nrows %d dst %p src %p\n", q->push_idx, row_size, nrows, dptr.dst, dptr.src); - q->push_idx = (q->push_idx + 1) & q->idx_mask; - return true; -} - -static inline dma_ptr dma_queue_pop(dma_queue * q) { - dma_ptr dptr = { NULL }; - - if (q->push_idx == q->pop_idx) { - return dptr; - } - - dma_descriptor_2d * desc = &q->desc[q->pop_idx]; - - // Wait for desc to complete - if (!desc->done) { - while (!desc->done) { - dmpoll(); - } - } - htp_trace_event_stop(q->trace, HTP_TRACE_EVT_DMA, q->pop_idx); - - dptr = q->dptr[q->pop_idx]; - - // FARF(ERROR, "dma-pop: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src); - q->pop_idx = (q->pop_idx + 1) & q->idx_mask; - return dptr; -} - -static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) { - dma_ptr dptr = { NULL }; - - if (q->push_idx == q->pop_idx) { - return dptr; - } - - dptr = q->dptr[q->pop_idx]; - - // FARF(ERROR, "dma-pop-nowait: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src); - q->pop_idx = (q->pop_idx + 1) & q->idx_mask; - return dptr; -} - -static inline bool dma_queue_empty(dma_queue * q) { - return q->push_idx == q->pop_idx; -} - -static inline uint32_t dma_queue_depth(dma_queue * q) { - return (q->push_idx - q->pop_idx) & q->idx_mask; -} - -static inline uint32_t dma_queue_capacity(dma_queue * q) { - return q->capacity; -} - -#if __HVX_ARCH__ < 75 - -// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535. -// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions. - -#define DMA_MAX_FIELD_VAL 65535u - -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // Fast path: everything fits in 16 bits - if (nrows == 0 || __builtin_expect( - row_size <= DMA_MAX_FIELD_VAL && - nrows <= DMA_MAX_FIELD_VAL && - src_stride <= DMA_MAX_FIELD_VAL && - dst_stride <= DMA_MAX_FIELD_VAL, 1)) { - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); - } - - // Contiguous block - // Use 1d DMA mode which supports sizes up to 24-bits (16MB) - if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { - size_t total = row_size * nrows; - return dma_queue_push_single_1d(q, dptr, total); - } - - // Stride overflow — fall back to row-by-row. - { - const uint8_t *src = (const uint8_t *) dptr.src; - uint8_t *dst = (uint8_t *) dptr.dst; - for (size_t r = 0; r < nrows; ++r) { - dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride); - if (!dma_queue_push_single_1d(q, p, row_size)) - return false; - if (r + 1 < nrows) - dma_queue_pop(q); - } - return true; - } -} - -#else // HVX_ARCH >= 75 - -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // On v75 and up we always use 2d 24-bit mode - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); -} - -#endif - -static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows); -} - -static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows); -} - -#define DMA_CACHE_MAX_SIZE 256U - -typedef struct { - uint8_t *base; - uint32_t line_size; - uint32_t capacity; - uint32_t src[DMA_CACHE_MAX_SIZE]; - uint16_t age[DMA_CACHE_MAX_SIZE]; -} dma_cache; - -static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity) -{ - c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; - c->base = base; - c->line_size = line_size; - - for (unsigned i=0; i < c->capacity; i++) { - c->src[i] = 0; - c->age[i] = 0; - } -} - -static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) -{ - uint32_t o_idx = 0; - uint16_t o_age = 0; - uint8_t * dst = 0; - - for (unsigned i=0; i < c->capacity; i++) { - if (c->src[i] == (uint32_t) src) { - c->age[i] = 0; - dst = c->base + (i * c->line_size); nrows = 0; // dummy dma - } else { - c->age[i]++; - if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; } - } - } - if (!dst) { - c->age[o_idx] = 0; - c->src[o_idx] = (uint32_t) src; - dst = c->base + o_idx * c->line_size; // normal nrows dma - return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); - } - - return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0); -} - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* HTP_DMA_H */ +#pragma once +#include "dma-queue.h" diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h index 8a37a4a066..a26961fc93 100644 --- a/ggml/src/ggml-hexagon/htp/hex-profile.h +++ b/ggml/src/ggml-hexagon/htp/hex-profile.h @@ -44,11 +44,11 @@ struct htp_thread_trace { }; static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { - if (tr && tr->events && tr->count < tr->max_events) { - uint32_t idx = tr->count; - tr->events[idx].id = id; - tr->events[idx].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); - tr->events[idx].cycles = (uint32_t) hex_get_cycles(); + if (tr->count < tr->max_events) { + uint32_t i = tr->count; + tr->events[i].id = id; + tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); + tr->events[i].cycles = (uint32_t) hex_get_cycles(); tr->count++; } } diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index 07930bef6e..93e87efcb4 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -30,21 +30,26 @@ static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, Q6_l2fetch_AP((void *) p, control); } -#define HEX_L2_LINE_SIZE 64 -#define HEX_L2_FLUSH_SIZE (128 * 1024) +static inline void hex_l2fetch_block(const void * addr, size_t size) { + if (size == 0) return; + const uint32_t width = 16384; // 16KB rows + const uint32_t height = (size + width - 1) / width; + hex_l2fetch(addr, width, width, height); +} + +#define HEX_L2_LINE_SIZE 128 +#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration) +#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) +#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) static inline void hex_l2flush(void * addr, size_t size) { - if (size > HEX_L2_FLUSH_SIZE) { - qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); - } else { - const uint32_t s = (uint32_t) addr; - const uint32_t e = s + size; - for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE * 4) { - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3); - } + const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1); + const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1); + for (uint32_t i = s; i < e; i += HEX_L2_BLOCK_SIZE) { + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0); + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1); + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2); + Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3); } } diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 740a8f87d6..a4efb1b004 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -1005,10 +1005,62 @@ static void transfer_activation_row_pair_fp32_to_fp16( } } +static void transfer_activation_row_pair_fp32_to_fp16_col_chunk( + __fp16 *restrict vtcm_dst, + const float *restrict row0, // offset by c_first + const float *restrict row1, // offset by c_first + uint32_t r, + uint32_t k_block, + uint32_t c_first, + uint32_t c_len, + uint32_t k_chunk_valid, + bool row0_valid, + bool row1_valid) { + + uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + uint32_t c = 0; + for (; c + 32 <= k_chunk_valid; c += 32) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < c_len) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + uint32_t rem = (k_chunk_valid > c) ? (k_chunk_valid - c) : 0; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } +} + static void transfer_activation_chunk_fp32_to_fp16_gathered( __fp16 *restrict vtcm_dst, const float *restrict src, uint32_t start_row, + uint32_t vtcm_start_row, uint32_t n_rows, uint32_t k_block, const struct mmid_row_mapping *matrix_rows, @@ -1029,8 +1081,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered( for (r = 0; r < n_rows_tiled; r += 2) { uint32_t r_idx0 = start_row + r + 0; uint32_t r_idx1 = start_row + r + 1; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0]; struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1]; @@ -1073,9 +1126,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered( } for (; r < n_rows_padded; r += 2) { - uint32_t r_idx0 = start_row + r; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx const bool row0_valid = (start_row + r + 0) < cne1; const bool row1_valid = (start_row + r + 1) < cne1; @@ -1135,6 +1188,7 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat( __fp16 *restrict vtcm_dst, const float *restrict src, uint32_t start_row, + uint32_t vtcm_start_row, uint32_t n_rows, uint32_t k_block, const struct mmid_row_mapping *matrix_rows, @@ -1152,8 +1206,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat( for (r = 0; r < n_rows_tiled; r += 2) { uint32_t r_idx0 = start_row + r + 0; uint32_t r_idx1 = start_row + r + 1; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0]; struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1]; @@ -1193,9 +1248,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat( } for (; r < n_rows_padded; r += 2) { - uint32_t r_idx0 = start_row + r; - uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx const bool row0_valid = (start_row + r + 0) < cne1; const bool row1_valid = (start_row + r + 1) < cne1; @@ -1253,6 +1308,7 @@ static void transfer_output_chunk_fp16_to_fp32_scattered( float *restrict dst, const __fp16 *restrict vtcm_src, uint32_t start_row, + uint32_t vtcm_start_row, uint32_t n_rows, uint32_t n_cols, const struct mmid_row_mapping *matrix_rows, @@ -1269,8 +1325,9 @@ static void transfer_output_chunk_fp16_to_fp32_scattered( for (size_t r = 0; r < n_rows; r += 2) { uint32_t r_idx0 = start_row + r + 0; uint32_t r_idx1 = start_row + r + 1; - const size_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; - const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile + uint32_t lr = vtcm_start_row + r; // vtcm-local row + const size_t r0 = (lr / HTP_MM_HMX_TILE_N_ROWS); + const size_t r1 = (lr % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; if (r_idx0 >= cne1) break; diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.c index 3add542bae..c369d3dd23 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.c +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c @@ -14,7 +14,7 @@ #define QURT_LOWEST_PRIO (254) -static inline void hmx_lock(struct hmx_queue *q) +static inline void hmx_lock(hmx_queue_t q) { if (!q->hmx_locked) { HAP_compute_res_hmx_lock(q->hap_rctx); @@ -22,7 +22,7 @@ static inline void hmx_lock(struct hmx_queue *q) } } -static inline void hmx_unlock(struct hmx_queue *q) +static inline void hmx_unlock(hmx_queue_t q) { if (q->hmx_locked) { HAP_compute_res_hmx_unlock(q->hap_rctx); @@ -30,7 +30,7 @@ static inline void hmx_unlock(struct hmx_queue *q) } } -static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { +static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { unsigned int ir = atomic_load(&q->idx_read); while (ir != atomic_load(&q->idx_write)) { @@ -61,7 +61,7 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { } static void hmx_queue_thread(void * arg) { - struct hmx_queue * q = (struct hmx_queue *) arg; + hmx_queue_t q = (hmx_queue_t) arg; FARF(HIGH, "hmx-queue-thread: started"); @@ -93,34 +93,41 @@ static void hmx_queue_thread(void * arg) { FARF(HIGH, "hmx-queue-thread: stopped"); } -struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) { +size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) { capacity = hex_ceil_pow2(capacity); + size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); + size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); + size_t size_stack = stack_size; + return size_q + size_desc + size_stack; +} + +size_t hmx_queue_alignof(void) { + return HEX_L2_LINE_SIZE; +} + +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) { + capacity = hex_ceil_pow2(capacity); + size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); + size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); + + uint8_t * block = (uint8_t *) ptr; + + hmx_queue_t q = (hmx_queue_t) block; block += size_q; + memset(q, 0, sizeof(struct hmx_queue_s)); - struct hmx_queue * q = (struct hmx_queue *) memalign(32, sizeof(struct hmx_queue)); - if (q == NULL) { - FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__); - return NULL; - } - memset(q, 0, sizeof(struct hmx_queue)); q->capacity = capacity; q->idx_mask = capacity - 1; q->hap_rctx = hap_rctx; + q->external_mem = true; - q->desc = (struct hmx_queue_desc *) memalign(64, capacity * sizeof(struct hmx_queue_desc)); - if (!q->desc) { - FARF(ERROR, "hmx-queue: failed to allocate HMX queue descriptors\n"); - return NULL; - } + q->desc = (struct hmx_queue_desc *) block; block += size_desc; memset(q->desc, 0, capacity * sizeof(struct hmx_queue_desc)); - const size_t stack_size = HMX_QUEUE_THREAD_STACK_SIZE; - q->stack = (unsigned char *) memalign(64, stack_size); - if (!q->stack) { - FARF(ERROR, "hmx-queue: thread stack allocation failed (%zu bytes)", stack_size); - return NULL; - } + q->stack = block; memset(q->stack, 0, stack_size); + q->trace = trace; + // Match caller thread priority (same pattern as worker-pool.c). int prio = qurt_thread_get_priority(qurt_thread_get_id()); if (prio < 1) { @@ -148,7 +155,7 @@ struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) { return q; } -void hmx_queue_delete(struct hmx_queue * q) { +void hmx_queue_free(hmx_queue_t q) { if (!q) { return; } @@ -160,8 +167,4 @@ void hmx_queue_delete(struct hmx_queue * q) { int status; qurt_thread_join(q->thread, &status); - - free(q->desc); - free(q->stack); - free(q); } diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.h index b176fa1796..c2b1859a28 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.h +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h @@ -17,8 +17,6 @@ extern "C" { #endif -#define HMX_QUEUE_THREAD_STACK_SIZE (16 * 1024) - #if __HVX_ARCH__ > 79 #define HMX_QUEUE_POLL_COUNT 2000 #else @@ -41,7 +39,7 @@ struct hmx_queue_desc { atomic_uint done; }; -struct hmx_queue { +struct hmx_queue_s { struct hmx_queue_desc * desc; atomic_uint idx_write; // updated by producer (push) atomic_uint idx_read; // updated by consumer (process) @@ -55,19 +53,24 @@ struct hmx_queue { uint32_t hap_rctx; bool hmx_locked; struct htp_thread_trace * trace; + bool external_mem; // memory owned externally }; -struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx); -void hmx_queue_delete(struct hmx_queue * q); +typedef struct hmx_queue_s * hmx_queue_t; + +size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size); +size_t hmx_queue_alignof(void); +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace); +void hmx_queue_free(hmx_queue_t q); static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) { struct hmx_queue_desc d = { func, data }; return d; } -static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) { +static inline bool hmx_queue_push(hmx_queue_t q, struct hmx_queue_desc d) { unsigned int ir = atomic_load(&q->idx_read); - unsigned int iw = q->idx_write; + unsigned int iw = atomic_load(&q->idx_write); if (((iw + 1) & q->idx_mask) == ir) { FARF(HIGH, "hmx-queue-push: queue is full\n"); @@ -87,25 +90,25 @@ static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) return true; } -static inline bool hmx_queue_signal(struct hmx_queue *q, enum hmx_queue_signal sig) { +static inline bool hmx_queue_signal(hmx_queue_t q, enum hmx_queue_signal sig) { return hmx_queue_push(q, hmx_queue_make_desc((hmx_queue_func) sig, NULL)); } -static inline bool hmx_queue_empty(struct hmx_queue * q) { - return q->idx_pop == q->idx_write; +static inline bool hmx_queue_empty(hmx_queue_t q) { + return q->idx_pop == atomic_load(&q->idx_write); } -static inline uint32_t hmx_queue_depth(struct hmx_queue * q) { - return (q->idx_read - q->idx_read) & q->idx_mask; +static inline uint32_t hmx_queue_depth(hmx_queue_t q) { + return (atomic_load(&q->idx_write) - atomic_load(&q->idx_read)) & q->idx_mask; } -static inline uint32_t hmx_queue_capacity(struct hmx_queue * q) { +static inline uint32_t hmx_queue_capacity(hmx_queue_t q) { return q->capacity; } -static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) { +static inline struct hmx_queue_desc hmx_queue_pop_one(hmx_queue_t q) { unsigned int ip = q->idx_pop; - unsigned int iw = q->idx_write; + unsigned int iw = atomic_load(&q->idx_write); struct hmx_queue_desc rd = { NULL, NULL }; if (ip == iw) { @@ -126,7 +129,7 @@ static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) { return rd; } -static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { +static inline struct hmx_queue_desc hmx_queue_pop(hmx_queue_t q) { while (1) { struct hmx_queue_desc d = hmx_queue_pop_one(q); @@ -138,15 +141,15 @@ static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { } } -static inline void hmx_queue_flush(struct hmx_queue * q) { +static inline void hmx_queue_flush(hmx_queue_t q) { while (hmx_queue_pop_one(q).func != NULL) ; } -static inline void hmx_queue_wakeup(struct hmx_queue * q) { +static inline void hmx_queue_wakeup(hmx_queue_t q) { hmx_queue_signal(q, HMX_QUEUE_WAKEUP); } -static inline void hmx_queue_suspend(struct hmx_queue *q) { +static inline void hmx_queue_suspend(hmx_queue_t q) { hmx_queue_signal(q, HMX_QUEUE_SUSPEND); } diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index e13103fb18..c4b60eb264 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -5,7 +5,8 @@ #include "hmx-queue.h" #include "htp-ops.h" #include "hex-profile.h" -#include "worker-pool.h" +#include "work-queue.h" +#include "hex-fastdiv.h" #include #include @@ -52,6 +53,9 @@ struct htp_ops_context { const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS]; }; + dma_queue ** src_dma[HTP_OP_MAX_INPUTS]; + dma_queue ** dst_dma[HTP_OP_MAX_OUTPUTS]; + // TODO convert these to an array struct htp_spad src0_spad; struct htp_spad src1_spad; @@ -65,11 +69,16 @@ struct htp_ops_context { // Main context for htp DSP backend struct htp_context { - dspqueue_t queue; - dma_queue * dma[HTP_MAX_NTHREADS]; + dspqueue_t dsp_queue; + struct htp_mmap mmap[HTP_MAX_MMAPS]; - worker_pool_context_t worker_pool; + dma_queue_t dma[HTP_MAX_NTHREADS]; + dma_queue_t dma_cached[HTP_MAX_NTHREADS]; + work_queue_t work_queue; + hmx_queue_t hmx_queue; + uint32_t n_threads; + struct fastdiv_values n_threads_div; int thread_id; int thread_prio; @@ -86,6 +95,7 @@ struct htp_context { atomic_bool vtcm_needs_release; uint64_t max_vmem; + uint32_t dirty_map[HTP_OP_MAX_TENSORS / 32]; // Persistent DDR scratchpad for MUL_MAT_ID mappings void * ddr_spad_base; @@ -93,7 +103,10 @@ struct htp_context { struct htp_ops_context octx; - struct hmx_queue * hmx_queue; // Async HMX queue for pipeline overlap + qurt_thread_t main_thread; + void * main_stack; + atomic_bool killed; + size_t footprint; }; int op_matmul(struct htp_ops_context * octx); diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index c9d0b3539a..1feaf2f999 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -108,8 +108,7 @@ enum htp_op_code { #define HTP_OP_MAX_KERN_PARAMS 32 #define HTP_OP_MAX_BUFS 16 -#define HTP_OP_MAX_REQS 256 -#define HTP_OP_MAX_TENSORS (HTP_OP_MAX_REQS * HTP_OP_MAX_INPUTS + HTP_OP_MAX_REQS) +#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16) #define HTP_OP_MAX_VMEM_DEFAULT (3355443200u) @@ -117,16 +116,18 @@ enum htp_op_code { enum htp_tensor_flags { HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights) - HTP_TENSOR_FLUSHED = (1U << 1) // Tensor buffer has been flushed (set by the NPU) + HTP_TENSOR_DIRTY = (1U << 1) // Tensor buffer is dirty and needs to be flushed }; // Tensor descriptor struct htp_tensor { uint32_t data; // Buffer offset in the messages, and data pointer on the NPU + uint32_t alias; // Index of the canonical tensor for this memory buffer uint32_t size; // Data size in bytes uint32_t flags; // Buffer / tensor flags - uint16_t type; // Data type + uint32_t type; // Data type uint16_t bi; // Buffer index + uint16_t ti; // Tensor index uint32_t ne[HTP_OP_MAX_DIMS]; // Number of elements uint32_t nb[HTP_OP_MAX_DIMS]; // Stride in bytes (see ggml.h ggml_tensor) }; @@ -169,6 +170,8 @@ enum htp_profiler_mode { enum htp_trace_event_id { HTP_TRACE_EVT_DMA = 0, + HTP_TRACE_EVT_L2FLUSH = 1, + HTP_TRACE_EVT_INIT = 2, HTP_TRACE_EVT_HVX_COMP = 20, HTP_TRACE_EVT_HVX_A_QUANT = 21, diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.c b/ggml/src/ggml-hexagon/htp/htp-tensor.c new file mode 100644 index 0000000000..3f028a7e45 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.c @@ -0,0 +1,204 @@ +#include "htp-tensor.h" + +#include +#include + +#include "hex-common.h" +#include "hex-utils.h" +#include "hex-fastdiv.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "work-queue.h" + +struct l2flush_task { + struct htp_thread_trace * trace; + uint32_t start; + uint32_t end; + uint32_t chunk_size; + uint32_t ti; +}; + +static void l2flush_thread_worker(unsigned int n, unsigned int i, void * data) { + struct l2flush_task * task = (struct l2flush_task *) data; + const uint32_t start = task->start; + const uint32_t end = task->end; + const uint32_t ti = task->ti; + const uint32_t chunk_size = task->chunk_size; + + const uint32_t thread_s = start + i * chunk_size; + if (thread_s >= end) { + return; + } + uint32_t thread_e = thread_s + chunk_size; + if (thread_e > end) { + thread_e = end; + } + + struct htp_thread_trace * tr = &task->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, ti); + hex_l2flush((void *) (uintptr_t) thread_s, thread_e - thread_s); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, ti); +} + +static void flush_all_dcache(struct htp_context * ctx) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + hex_l2fetch_block(ctx, ctx->footprint); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS); +} + +static void flush_tensor_range(struct htp_context * ctx, const struct htp_tensor * t) { + struct htp_thread_trace * tr = &ctx->trace[0]; + + if (t->size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_task task; + task.start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE); + task.end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE); + task.ti = t->ti; + task.trace = ctx->trace; + + const uint32_t total_size = task.end - task.start; + const uint32_t n_blocks = (total_size + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + const uint32_t blocks_per_thread = fastdiv(n_blocks + ctx->n_threads - 1, &ctx->n_threads_div); + task.chunk_size = blocks_per_thread * HEX_L2_BLOCK_SIZE; + + work_queue_run(ctx->work_queue, l2flush_thread_worker, &task, ctx->n_threads); + } else { + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + hex_l2flush((void *) t->data, t->size); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + } + + htp_tensor_make_clean(t, ctx->dirty_map); +} + +void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t) { + if (!bitmap_test(ctx->dirty_map, t->ti)) { + return; + } + + if (t->size > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + return; + } + + flush_tensor_range(ctx, t); +} + +// One dirty tensor's line-aligned range, placed in the flattened global block space. +struct l2flush_range { + uint32_t start; // line-aligned start address + uint32_t end; // line-aligned end address + uint32_t block_first; // global block index of this range's first block + uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial) +}; + +struct l2flush_multi_task { + struct htp_thread_trace * trace; + struct l2flush_range ranges[HTP_OP_MAX_INPUTS]; + uint32_t n_ranges; + uint32_t total_blocks; + uint32_t blocks_per_thread; +}; + +static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct l2flush_multi_task * task = (struct l2flush_multi_task *) data; + + const uint32_t gb_first = i * task->blocks_per_thread; + uint32_t gb_last = gb_first + task->blocks_per_thread; + if (gb_last > task->total_blocks) { + gb_last = task->total_blocks; + } + if (gb_first >= gb_last) { + return; + } + + struct htp_thread_trace * tr = &task->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); + + for (uint32_t r = 0; r < task->n_ranges; r++) { + const struct l2flush_range * rg = &task->ranges[r]; + const uint32_t rb_first = rg->block_first; + const uint32_t rb_last = rg->block_first + rg->n_blocks; + + const uint32_t lo = gb_first > rb_first ? gb_first : rb_first; + const uint32_t hi = gb_last < rb_last ? gb_last : rb_last; + if (lo >= hi) { + continue; + } + + const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE; + uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE; + if (e > rg->end) { + e = rg->end; + } + hex_l2flush((void *) (uintptr_t) s, e - s); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); +} + +void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { + uint64_t total_dirty = 0; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (t && bitmap_test(ctx->dirty_map, t->ti)) { + total_dirty += t->size; + } + } + + if (total_dirty == 0) { + return; + } + + if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + return; + } + + // Aggregate is small enough to walk. Thread it across all dirty ranges at once + // when it is worth the dispatch, otherwise flush sequentially. + if (total_dirty > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_multi_task task; + task.trace = ctx->trace; + task.n_ranges = 0; + + uint32_t block_acc = 0; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (!t || !bitmap_test(ctx->dirty_map, t->ti)) { + continue; + } + // Clear as we go: dedups a tensor passed as multiple srcs (e.g. mul(x,x)). + htp_tensor_make_clean(t, ctx->dirty_map); + + struct l2flush_range * rg = &task.ranges[task.n_ranges++]; + rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE); + rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE); + rg->block_first = block_acc; + rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + block_acc += rg->n_blocks; + } + + task.total_blocks = block_acc; + task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + + work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + return; + } + + struct htp_thread_trace * tr = &ctx->trace[0]; + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (!t || !bitmap_test(ctx->dirty_map, t->ti)) { + continue; + } + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + hex_l2flush((void *) t->data, t->size); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti); + htp_tensor_make_clean(t, ctx->dirty_map); + } +} diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h new file mode 100644 index 0000000000..e503bb90f5 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -0,0 +1,36 @@ +#ifndef HTP_TENSOR_H +#define HTP_TENSOR_H + +#include +#include "htp-ops.h" +#include "hex-bitmap.h" + +static inline struct htp_tensor * htp_tensor_alias(const struct htp_tensor * t) { + return (struct htp_tensor *) (uintptr_t) t->alias; +} + +static inline void * htp_tensor_data(const struct htp_tensor * t) { + return (void *) (uintptr_t) t->data; +} + +static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) { + return (uint32_t *) &t->flags; +} + +static inline void htp_tensor_make_dirty(const struct htp_tensor * t, uint32_t * dirty_map) { + struct htp_tensor * curr = (struct htp_tensor *) t; + do { + bitmap_set(dirty_map, curr->ti); + curr = htp_tensor_alias(curr); + } while (curr != t); +} + +static inline void htp_tensor_make_clean(const struct htp_tensor * t, uint32_t * dirty_map) { + bitmap_clear(dirty_map, t->ti); +} + +struct htp_context; +void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t); +void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); + +#endif // HTP_TENSOR_H diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index d971b60f3a..b53bb5024e 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -25,112 +25,44 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-bitmap.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #include "htp_iface.h" -#include "worker-pool.h" +#include "work-queue.h" +#include "hex-profile.h" + +#define HMX_QUEUE_CAPACITY 16 +#define HMX_QUEUE_STACK_SIZE 16384 +#define WORK_QUEUE_CAPACITY 16 +#define WORK_QUEUE_STACK_SIZE 16384 +#define MAIN_THREAD_STACK_SIZE 32768 + +_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS, + "work-queue thread cap must be >= HTP_MAX_NTHREADS"); + +struct htp_handle { + struct htp_context * ctx; +}; AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) { - struct htp_context * ctx; - int err = 0; - - ctx = calloc(1, sizeof(*ctx)); - if (ctx == NULL) { + (void) uri; + struct htp_handle * h = calloc(1, sizeof(*h)); + if (h == NULL) { return AEE_ENOMEMORY; } - // Use the context structure as the handle - *handle = (remote_handle64) ctx; - - // Enable FARF logs - HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0); - - // Set client class - { - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_apptype; - request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; - - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; - } - } - - { - HAP_power_request_t request; - memset(&request, 0, sizeof(request)); - - request.type = HAP_power_set_DCVS_v3; - request.dcvs_v3.set_dcvs_enable = TRUE; - request.dcvs_v3.dcvs_enable = FALSE; - request.dcvs_v3.set_bus_params = TRUE; - request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.set_core_params = TRUE; - request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.set_sleep_disable = TRUE; - request.dcvs_v3.sleep_disable = TRUE; - -#if (__HEXAGON_ARCH__ >= 79) - HAP_set_dcvs_v3_protected_bus_corners(&request, 1); -#endif - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; - } - - memset(&request, 0, sizeof(request)); - request.type = HAP_power_set_HVX; - request.hvx.power_up = TRUE; - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; - } - } - -#if __HVX_ARCH__ >= 75 - { - // Power on HMX and set HMX clock - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_HMX_v2; - request.hmx_v2.set_power = TRUE; - request.hmx_v2.power_up = TRUE; - request.hmx_v2.set_clock = TRUE; - request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX; - request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH; - FARF(ALWAYS, "Setting HMX clock\n"); - err = HAP_power_set((void *) ctx, &request); - if (err != AEE_SUCCESS) { - FARF(ERROR, "ggml-hex: error setting HMX clock."); - return err; - } - } -#else - { - // Power on HMX - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_HMX; - request.hmx.power_up = TRUE; - FARF(ALWAYS, "Powering HMX on\n"); - err = HAP_power_set((void *) ctx, &request); - if (err != AEE_SUCCESS) { - FARF(ERROR, "ggml-hex: error powering on HMX."); - return err; - } - } -#endif - + *handle = (remote_handle64) h; return AEE_SUCCESS; } AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { + return AEE_EBADPARM; + } + int err = enable ? HAP_user_etm_enable() : HAP_user_etm_disable(); if (err) { if (err == AEE_EVERSIONNOTSUPPORT) { @@ -143,10 +75,11 @@ AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) { } AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_iface_pmu_conf* pmu_conf) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; if (mode == HTP_PROF_PMU) { const uint32_t* events = pmu_conf->events; @@ -179,48 +112,55 @@ AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_if } AEEResult htp_iface_close(remote_handle64 handle) { - struct htp_context * ctx = (struct htp_context *) handle; - - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { return AEE_EBADPARM; } - if (ctx->queue) { - FARF(ERROR, "Closing handle with queue still open"); - return AEE_EITEMBUSY; - } - - // release the mmaps (if any) - for (uint32_t i=0; immap[i].size) { -#if __HVX_ARCH__ > 73 - HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size); -#else - HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); -#endif - ctx->mmap[i].size = 0; - ctx->mmap[i].base = NULL; - ctx->mmap[i].fd = -1; + struct htp_context * ctx = h->ctx; + if (ctx) { + if (ctx->dsp_queue) { + FARF(ERROR, "Closing handle with queue still open"); + return AEE_EITEMBUSY; } + + // release the mmaps (if any) + for (uint32_t i=0; immap[i].size) { +#if __HVX_ARCH__ > 73 + HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size); +#else + HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); +#endif + ctx->mmap[i].size = 0; + ctx->mmap[i].base = NULL; + ctx->mmap[i].fd = -1; + } + } + + if (ctx->profiler) { + qurt_pmu_enable(1); + } + + if (ctx->etm) { + HAP_user_etm_disable(); + } + + // Free the unified block (ctx is the base address of the block) + free(ctx); + h->ctx = NULL; } - if (ctx->profiler) { - qurt_pmu_enable(1); - } - - if (ctx->etm) { - HAP_user_etm_disable(); - } - - free(ctx); + free(h); return AEE_SUCCESS; } AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; // See if we already have this mapping for (uint32_t i=0; ictx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; for (uint32_t i=0; immap[i]; @@ -358,56 +299,44 @@ static void vtcm_free(struct htp_context * ctx) { } } +static void htp_main_thread(void * context); static void htp_packet_callback(dspqueue_t queue, int error, void * context); static void htp_error_callback(dspqueue_t queue, int error, void * context); AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem) { - struct htp_context * ctx = (struct htp_context *) handle; - - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { return AEE_EBADPARM; } - if (ctx->queue) { + if (h->ctx) { FARF(ERROR, "Queue already open"); return AEE_EITEMBUSY; } - // Import queue created on the CPU - int err = dspqueue_import(dsp_queue_id, // Queue ID from dspqueue_export - htp_packet_callback, // Packet callback - htp_error_callback, // Error callback; no errors expected on the DSP - (void *) ctx, // Callback context - &ctx->queue); + // Cache the original FastRPC thread priority, then calculate compute priority + int fastrpc_tid = qurt_thread_get_id(); + int fastrpc_prio = qurt_thread_get_priority(fastrpc_tid); + int main_prio = fastrpc_prio - 10; + if (main_prio < 1) main_prio = 1; + + dspqueue_t dsp_queue = NULL; + bool use_callbacks = false; + + // Import queue with NULL callbacks to avoid starting dspueue internal threads + int err = dspqueue_import(dsp_queue_id, NULL, NULL, (void *) h, &dsp_queue); + if (err == AEE_EBADPARM) { + // Fallback for devices that don't support NULL callbacks + FARF(HIGH, "dspqueue import with NULL callbacks failed, trying with callbacks"); + use_callbacks = true; + err = dspqueue_import(dsp_queue_id, htp_packet_callback, htp_error_callback, (void *) h, &dsp_queue); + } + if (err) { FARF(ERROR, "Queue import failed with 0x%08x", (unsigned) err); return err; } - ctx->max_vmem = max_vmem; - ctx->thread_id = qurt_thread_get_id(); - ctx->thread_prio = qurt_thread_get_priority(ctx->thread_id); - - // allocate VTCM - err = vtcm_alloc(ctx); - if (err != AEE_SUCCESS) { - FARF(ERROR, "Unable to allocate VTCM"); - return AEE_ENOMEMORY; - } - - ctx->hmx_enabled = n_hmx; - ctx->hmx_queue = NULL; - if (n_hmx) { - ctx->hmx_queue = hmx_queue_create(16, ctx->vtcm_rctx); - if (ctx->hmx_queue) { - ctx->hmx_queue->trace = &ctx->trace[HTP_MAX_NTHREADS]; - } else { - FARF(ERROR, "hmx-queue-create failed"); - ctx->hmx_enabled = false; - } - } - FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); - qurt_sysenv_max_hthreads_t hw_threads; qurt_sysenv_get_max_hw_threads(&hw_threads); uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; @@ -422,27 +351,216 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp n_hvx = HTP_MAX_NTHREADS; } - ctx->n_threads = n_hvx; - for (int i = 0; i < ctx->n_threads; i++) { - ctx->dma[i] = dma_queue_create(256); // queue depth - if (ctx->dma[i]) { - ctx->dma[i]->trace = &ctx->trace[i]; + // layout segments of our contiguous block + + // 1. htp_context : sits at the base (block is 4K-aligned via memalign below) + size_t offset = sizeof(struct htp_context); + + // 2. main_stack + size_t offset_main_stack = 0; + size_t size_main_stack = 0; + if (!use_callbacks) { + offset_main_stack = hex_align_up(offset, 4096); + size_main_stack = MAIN_THREAD_STACK_SIZE; + offset = offset_main_stack + size_main_stack; + } + + // 3. work_queue + size_t wq_align = work_queue_alignof(); + size_t offset_wq = hex_align_up(offset, wq_align); + size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + offset = offset_wq + size_wq; + + // 4. dma_queue + size_t dma_align = dma_queue_alignof(); + size_t offset_dma = hex_align_up(offset, dma_align); + size_t size_dma = 0; + for (uint32_t i = 0; i < n_hvx; i++) { + size_dma = hex_align_up(size_dma, dma_queue_alignof()); + size_dma += dma_queue_sizeof(256); + size_dma = hex_align_up(size_dma, dma_queue_alignof()); + size_dma += dma_queue_alias_sizeof(); + } + offset = offset_dma + size_dma; + + // 5. hmx_queue + size_t offset_hmx = 0; + size_t size_hmx = 0; + if (n_hmx) { + size_t hmx_align = hmx_queue_alignof(); + offset_hmx = hex_align_up(offset, hmx_align); + size_hmx = hmx_queue_sizeof(HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE); + offset = offset_hmx + size_hmx; + } + + size_t footprint = hex_align_up(offset, 128); + + void * block = memalign(4096, footprint); + if (!block) { + FARF(ERROR, "Unable to allocate unified block of size %zu\n", footprint); + dspqueue_close(dsp_queue); + return AEE_ENOMEMORY; + } + memset(block, 0, footprint); + + h->ctx = (struct htp_context *) block; + struct htp_context * ctx = h->ctx; + ctx->footprint = footprint; + + ctx->thread_id = fastrpc_tid; + ctx->thread_prio = main_prio; + ctx->max_vmem = max_vmem; + ctx->dsp_queue = dsp_queue; + + err = vtcm_alloc(ctx); + if (err != AEE_SUCCESS) { + FARF(ERROR, "Unable to allocate VTCM"); + htp_iface_stop(handle); + return AEE_ENOMEMORY; + } + + HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0); + + // Set client class + { + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_apptype; + request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; + + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; } } + // DCVS setup + { + HAP_power_request_t request; + memset(&request, 0, sizeof(request)); + + request.type = HAP_power_set_DCVS_v3; + request.dcvs_v3.set_dcvs_enable = TRUE; + request.dcvs_v3.dcvs_enable = FALSE; + request.dcvs_v3.set_bus_params = TRUE; + request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.set_core_params = TRUE; + request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX; + request.dcvs_v3.set_sleep_disable = TRUE; + request.dcvs_v3.sleep_disable = TRUE; + +#if (__HEXAGON_ARCH__ >= 79) + HAP_set_dcvs_v3_protected_bus_corners(&request, 1); +#endif + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; + } + + memset(&request, 0, sizeof(request)); + request.type = HAP_power_set_HVX; + request.hvx.power_up = TRUE; + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; + } + } + +#if __HVX_ARCH__ >= 75 + { + // Power on HMX and set HMX clock + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_HMX_v2; + request.hmx_v2.set_power = TRUE; + request.hmx_v2.power_up = TRUE; + request.hmx_v2.set_clock = TRUE; + request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX; + request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX; + request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX; + request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH; + FARF(ALWAYS, "Setting HMX clock\n"); + err = HAP_power_set((void *) ctx, &request); + if (err != AEE_SUCCESS) { + FARF(ERROR, "ggml-hex: error setting HMX clock."); + htp_iface_stop(handle); + return err; + } + } +#else + { + // Power on HMX + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_HMX; + request.hmx.power_up = TRUE; + FARF(ALWAYS, "Powering HMX on\n"); + err = HAP_power_set((void *) ctx, &request); + if (err != AEE_SUCCESS) { + FARF(ERROR, "ggml-hex: error powering on HMX."); + htp_iface_stop(handle); + return err; + } + } +#endif + + ctx->hmx_enabled = n_hmx; + ctx->hmx_queue = NULL; + if (n_hmx) { + void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); + ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]); + } + FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); + + ctx->n_threads = n_hvx; + ctx->n_threads_div = init_fastdiv_values(ctx->n_threads); + + // Initialize DMA queues + uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma); + size_t size_dma_q = dma_queue_sizeof(256); + size_t size_dma_alias = dma_queue_alias_sizeof(); + + for (int i = 0; i < ctx->n_threads; i++) { + dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof()); + ctx->dma_cached[i] = dma_queue_init(dma_ptr_curr, 256, (uintptr_t) ctx->vtcm_base, ctx->vtcm_size, &ctx->trace[i]); + dma_ptr_curr += size_dma_q; + + dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof()); + ctx->dma[i] = dma_queue_alias_init(dma_ptr_curr, ctx->dma_cached[i], 1); + dma_ptr_curr += size_dma_alias; + } + ctx->ddr_spad_size = 512 * 1024; // 512 KB ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size); - // init worker pool - err = worker_pool_init(&ctx->worker_pool, n_hvx); - if (err != AEE_SUCCESS) { - FARF(ERROR, "Unable to create worker pool"); - if (ctx->ddr_spad_base) { - free(ctx->ddr_spad_base); - ctx->ddr_spad_base = NULL; - ctx->ddr_spad_size = 0; + void * wq_ptr = (void *) ((uintptr_t) block + offset_wq); + ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + + ctx->main_stack = NULL; + ctx->main_thread = 0; + atomic_store(&ctx->killed, false); + + if (!use_callbacks) { + // Start main compute thread + ctx->main_stack = (void *) ((uintptr_t) block + offset_main_stack); + + qurt_thread_attr_t attr; + qurt_thread_attr_init(&attr); + qurt_thread_attr_set_stack_addr(&attr, ctx->main_stack); + qurt_thread_attr_set_stack_size(&attr, size_main_stack); + qurt_thread_attr_set_priority(&attr, main_prio); + qurt_thread_attr_set_name(&attr, "htp-main"); + + int err_thread = qurt_thread_create(&ctx->main_thread, &attr, htp_main_thread, ctx); + if (err_thread) { + FARF(ERROR, "Unable to create htp main thread: %d", err_thread); + htp_iface_stop(handle); + return AEE_ENOMEMORY; } - return err; } FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n", @@ -452,35 +570,34 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp } AEEResult htp_iface_stop(remote_handle64 handle) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; - if (!ctx->queue) { - FARF(ERROR, "Queue not open"); - return AEE_EBADSTATE; + if (ctx->main_thread) { + atomic_store(&ctx->killed, true); + int status; + (void) qurt_thread_join(ctx->main_thread, &status); + ctx->main_thread = 0; } - // Close queue. dspqueue_close() will also wait for callbacks to finish. - int err = dspqueue_close(ctx->queue); - ctx->queue = NULL; + int err = dspqueue_close(ctx->dsp_queue); ctx->dsp_queue = NULL; if (err != 0) { FARF(ERROR, "Queue close failed with 0x%08x", (unsigned) err); return err; } - if (ctx->worker_pool) { - // Release worker pool - worker_pool_release(&ctx->worker_pool); - } + work_queue_free(ctx->work_queue); for (int i = 0; i < ctx->n_threads; i++) { - dma_queue_delete(ctx->dma[i]); + dma_queue_alias_free(ctx->dma[i]); + dma_queue_free(ctx->dma_cached[i]); } if (ctx->hmx_queue) { - hmx_queue_delete(ctx->hmx_queue); + hmx_queue_free(ctx->hmx_queue); ctx->hmx_queue = NULL; } ctx->hmx_enabled = false; @@ -493,6 +610,9 @@ AEEResult htp_iface_stop(remote_handle64 handle) { ctx->ddr_spad_size = 0; } + free(ctx); + h->ctx = NULL; + return AEE_SUCCESS; } @@ -671,8 +791,6 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_INVALID: break; - - // No default to catch missing cases } FARF(ERROR, "Unknown Op %u", octx->op); @@ -778,12 +896,14 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin } } -static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t idx, struct htp_tensor *t) { +static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t idx, struct htp_tensor *t) { uint32_t offset = t->data; uint32_t size = t->size; uint32_t bi = t->bi; + uint32_t alias = t->alias; - t->data = bufs[bi].base + offset; // update data to the actual pointer + t->data = (uint32_t) (bufs[bi].base + offset); // update data to the actual pointer + t->alias = (uint32_t) (tens + alias); // update alias to the actual pointer FARF(HIGH, "prep-tensor #%u: bi %u offset %u size %u data %p : %u:%u:%u:%u", idx, t->bi, offset, t->size, (void*) t->data, t->ne[0], t->ne[1], t->ne[3], t->ne[3]); @@ -791,7 +911,7 @@ static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t n_tens) { for (uint32_t i=0; i < n_tens; i++) { - prep_tensor(ctx, bufs, i, tens + i); + prep_tensor(ctx, bufs, tens, i, tens + i); } } @@ -805,29 +925,36 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u // Prep input tensors for (uint32_t i=0; isrc[i] == 0xffff ? NULL : tens + op->src[i]; - - octx->src[i] = src; - if (!src) continue; - - if (!(src->flags & HTP_TENSOR_FLUSHED) && (src->flags & HTP_TENSOR_COMPUTE)) { - // flush compute buffers on input - hex_l2flush((void *) src->data, src->size); + uint16_t src_idx = op->src[i]; + if (src_idx == 0xffff) { + octx->src[i] = NULL; + octx->src_dma[i] = NULL; + continue; } + struct htp_tensor *src = tens + src_idx; + octx->src[i] = src; + octx->src_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma; + FARF(HIGH, "prep-src #%u: data %p size %u : %u:%u:%u:%u", op->src[i], (void*) src->data, src->size, src->ne[0], src->ne[1], src->ne[3], src->ne[3]); } + htp_tensor_flush_all(octx->ctx, octx->src, HTP_OP_MAX_INPUTS); + // Prep output tensors for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { uint16_t dst_idx = op->dst[i]; if (dst_idx == 0xffff) { - octx->dsts[i] = NULL; + octx->dsts[i] = NULL; + octx->dst_dma[i] = NULL; continue; } struct htp_tensor *dst = tens + dst_idx; - octx->dsts[i] = dst; + octx->dsts[i] = dst; + octx->dst_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma; + + htp_tensor_make_dirty(dst, octx->ctx->dirty_map); FARF(HIGH, "prep-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, dst_idx, (void*) dst->data, dst->size, dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); @@ -841,34 +968,134 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u octx->src3_spad.src = NULL; octx->dst_spad.src = NULL; - // flush buffers on output - for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { - if (octx->dsts[i]) { - struct htp_tensor *dst = (struct htp_tensor *)octx->dsts[i]; - hex_l2flush((void *) dst->data, dst->size); - dst->flags |= HTP_TENSOR_FLUSHED; - - FARF(HIGH, "post-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, op->dst[i], (void*) dst->data, dst->size, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); - } - } - return status; } +static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) { + dspqueue_t queue = ctx->dsp_queue; + int err; + + const uint32_t n_bufs = req->n_bufs; + const uint32_t n_tens = req->n_tensors; + const uint32_t n_ops = req->n_ops; + + const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs; + const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; + const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; + const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; + const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc); + + if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) { + FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size); + return; + } + + FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id, + n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size); + + // Clean cache at the start of the batch + // We cant trace this part because the trace buffer is setup later + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + hex_l2fetch_block(ctx, ctx->footprint); + bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS); + + // Setup descriptor pointers + uint8_t * m_ptr = dbuf->ptr; + struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size; + struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size; + struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size; + struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr; + + prep_op_bufs(ctx, bufs, n_bufs); + prep_tensors(ctx, bufs, tens, n_tens); + + struct htp_ops_context *octx = &ctx->octx; + memset(octx, 0, sizeof(*octx)); + octx->n_threads = ctx->n_threads; + octx->ctx = ctx; + + memset(ctx->trace, 0, sizeof(ctx->trace)); + if (ctx->profiler == HTP_PROF_TRACE) { + struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); + for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + ctx->trace[t].events = &trace_events[t * req->n_traces]; + ctx->trace[t].max_events = req->n_traces; + } + } + + work_queue_wakeup(ctx->work_queue); + if (ctx->hmx_queue) { + hmx_queue_wakeup(ctx->hmx_queue); + } + + int op_status = HTP_STATUS_OK; + for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) { + struct profile_data prof; + + profile_start(ctx->profiler, &prof); + + op_status = proc_op_req(octx, tens, i, &ops[i]); + + profile_stop(ctx->profiler, &prof); + + if (ctx->profiler) { + pds[i].opcode = ops[i].opcode; + pds[i].usecs = prof.usecs; + pds[i].cycles_start = prof.cycles_start; + pds[i].cycles_stop = prof.cycles_stop; + for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) { + pds[i].pmu[j] = prof.pmu_counters[j]; + } + } + } + + if (ctx->hmx_queue) { + hmx_queue_suspend(ctx->hmx_queue); + hmx_queue_flush(ctx->hmx_queue); + } + work_queue_suspend(ctx->work_queue); + + struct htp_opbatch_rsp rsp; + memset(&rsp, 0, sizeof(rsp)); + rsp.id = req->id; + rsp.status = op_status; + rsp.n_bufs = n_bufs; + rsp.n_tensors = n_tens; + rsp.n_ops = n_ops; + + if (ctx->profiler == HTP_PROF_TRACE) { + for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + rsp.n_traces[t] = ctx->trace[t].count; + } + } + + struct dspqueue_buffer write_dbuf = *dbuf; + write_dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; + + // Flush remaining dirty tensors at the end of the batch + htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + + err = dspqueue_write(queue, 0, 1, &write_dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE); + if (err != 0) { + FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err); + } +} + +#define DSPQUEUE_READ_TIMEOUT_USEC 5000 #define DSPQUEUE_POLL_TIMEOUT_USEC 100 #define DSPQUEUE_POLL_COUNT 100 -static void htp_packet_callback(dspqueue_t queue, int error, void * context) { - struct htp_context * ctx = (struct htp_context *) context; - +static void process_ops(struct htp_context * ctx) { + dspqueue_t queue = ctx->dsp_queue; int err; uint32_t poll_count = DSPQUEUE_POLL_COUNT; vtcm_acquire(ctx); - while (!ctx->vtcm_needs_release) { + while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) { struct htp_opbatch_req req; uint32_t r_size = sizeof(req); @@ -898,111 +1125,41 @@ static void htp_packet_callback(dspqueue_t queue, int error, void * context) { // Reset poll count for valid requests poll_count = DSPQUEUE_POLL_COUNT; - const uint32_t n_bufs = req.n_bufs; - const uint32_t n_tens = req.n_tensors; - const uint32_t n_ops = req.n_ops; - - const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs; - const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; - const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; - const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; - const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(struct htp_trace_desc); - - if (dbuf.size < b_size + t_size + o_size + p_size + tr_size) { - FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf.size, b_size + t_size + o_size + p_size + tr_size); - break; - } - - FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req.id, - n_bufs, n_tens, n_ops, req.n_traces, dbuf.size, b_size, t_size, o_size); - - // Setup descriptor pointers - uint8_t * m_ptr = dbuf.ptr; - struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size; - struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size; - struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size; - struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr; - - prep_op_bufs(ctx, bufs, n_bufs); - prep_tensors(ctx, bufs, tens, n_tens); - - struct htp_ops_context *octx = &ctx->octx; - memset(octx, 0, sizeof(*octx)); - octx->n_threads = ctx->n_threads; - octx->ctx = ctx; - - if (ctx->profiler == HTP_PROF_TRACE) { - memset(ctx->trace, 0, sizeof(ctx->trace)); - struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { - ctx->trace[t].events = &trace_events[t * req.n_traces]; - ctx->trace[t].max_events = req.n_traces; - } - } else { - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { - ctx->trace[t].events = NULL; - ctx->trace[t].max_events = 0; - } - } - - int op_status = HTP_STATUS_OK; - uint32_t op_wakeup = n_ops / 2; // half-way throgh the batch - - hmx_queue_wakeup(ctx->hmx_queue); - - for (uint32_t i=0; i < n_ops; i++) { - struct profile_data prof; - - if (i == op_wakeup) { - dspqueue_write_early_wakeup_noblock(queue, 0, 0); - } - - profile_start(ctx->profiler, &prof); - - op_status = proc_op_req(octx, tens, i, &ops[i]); - - profile_stop(ctx->profiler, &prof); - - if (op_status != HTP_STATUS_OK) { - break; - } - - if (ctx->profiler) { - pds[i].opcode = ops[i].opcode; - pds[i].usecs = prof.usecs; - pds[i].cycles_start = prof.cycles_start; - pds[i].cycles_stop = prof.cycles_stop; - for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) { - pds[i].pmu[j] = prof.pmu_counters[j]; - } - } - } - - hmx_queue_suspend(ctx->hmx_queue); - - struct htp_opbatch_rsp rsp; - rsp.id = req.id; - rsp.status = op_status; - rsp.n_bufs = n_bufs; - rsp.n_tensors = n_tens; - rsp.n_ops = n_ops; - memset(rsp.pad, 0, sizeof(rsp.pad)); - if (ctx->profiler == HTP_PROF_TRACE) { - for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { - rsp.n_traces[t] = ctx->trace[t].count; - } - } else { - memset(rsp.n_traces, 0, sizeof(rsp.n_traces)); - } - - dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; - - err = dspqueue_write(queue, 0, 1, &dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE); - if (err != 0) { - FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err); - break; - } + process_opbatch(ctx, &req, &dbuf); } vtcm_release(ctx); } + +static void htp_packet_callback(dspqueue_t queue, int error, void * context) { + (void) queue; + (void) error; + struct htp_handle * h = (struct htp_handle *) context; + if (h && h->ctx) { + process_ops(h->ctx); + } +} + +static void htp_main_thread(void * context) { + struct htp_context * ctx = (struct htp_context *) context; + + FARF(HIGH, "htp-main-thread: started"); + + while (!atomic_load(&ctx->killed)) { + uint32_t flags = 0; + uint32_t num_buffers = 0; + uint32_t message_length = 0; + + int err = dspqueue_peek(ctx->dsp_queue, &flags, &num_buffers, &message_length, 50000); + if (err == 0) { + process_ops(ctx); + } else if (err == AEE_EWOULDBLOCK || err == AEE_EEXPIRED) { + continue; + } else { + FARF(ERROR, "dspqueue_peek failed: 0x%08x", (unsigned) err); + break; + } + } + + FARF(HIGH, "htp-main-thread: stopped"); +} diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 1683131a81..0611aa9826 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -92,10 +92,10 @@ struct htp_mm_context { // Per thread quant tasks // Precomputed block-parallel quantization values worker_callback_t quant_task_func; - uint32_t quant_ib_first[MAX_NUM_WORKERS]; - uint32_t quant_ib_last[MAX_NUM_WORKERS]; - uint32_t quant_r[MAX_NUM_WORKERS]; - uint32_t quant_c[MAX_NUM_WORKERS]; + uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS]; uint32_t n_quant_tasks; uint32_t n_quant_rows_per_thread; atomic_uint quant_barrier; @@ -254,7 +254,7 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { return; } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0_start); const uint32_t blck_0 = 64; @@ -309,7 +309,7 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ const uint32_t n_prefetch = kparams->n_prefetch; \ @@ -410,7 +410,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ const uint32_t n_prefetch = kparams->n_prefetch; \ @@ -523,7 +523,7 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith; \ uint8_t * restrict src1_data = mmctx->vtcm_src1; \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ const uint32_t n_prefetch = kparams->n_prefetch; \ @@ -699,7 +699,7 @@ static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \ uint8_t * restrict src1_data = mmctx->vtcm_src1; \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \ const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \ @@ -820,7 +820,7 @@ static void name(unsigned int nth, unsigned int ith, void * data) { return; \ } \ \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ \ uint8_t * restrict dst = mmctx->vtcm_src1; \ @@ -846,7 +846,7 @@ QUANTIZE_IMPL(quantize_f16_f16_flat, "quantize-f16-f16", quantize_f16_f static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = octx->src[1]; @@ -870,7 +870,7 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) { struct htp_mm_context * mmctx = data; struct htp_ops_context * octx = mmctx->octx; - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); const struct htp_tensor * src = octx->src[1]; @@ -944,7 +944,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -1040,7 +1040,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_start_row = src0_nrows_per_thread * ith; const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const size_t dst_row_size = nb1; const size_t src0_row_size = nb01; @@ -1155,7 +1155,7 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { return; } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1244,7 +1244,7 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { return; } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const uint32_t n_prefetch = kparams->n_prefetch; @@ -1338,6 +1338,9 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type static int hvx_mm_matmul(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; @@ -1557,9 +1560,6 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->vtcm_src0_stride = src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; - if (need_quant) { mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; @@ -1570,8 +1570,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->n_quant_tasks = 0; } - const uint32_t n_matmul_jobs = octx->n_threads; - worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, octx->n_threads); return HTP_STATUS_OK; } @@ -1874,7 +1875,7 @@ static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) { #define DEQUANTIZE_WORKER_LOOP_IMPL(SUFFIX) \ static void dequantize_tiled_worker_loop_##SUFFIX(unsigned int n, unsigned int i, void *data) { \ tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; \ - struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; \ + struct htp_thread_trace * tr = &state->traces[i]; \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); \ for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { \ int start = task_id * state->n_tiles_per_task; \ @@ -1892,7 +1893,7 @@ DEQUANTIZE_WORKER_LOOP_IMPL(q8_0) static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) { tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; - struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; + struct htp_thread_trace * tr = &state->traces[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { int start = task_id * state->n_tiles_per_task; @@ -1905,7 +1906,7 @@ static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data) { tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; - struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; + struct htp_thread_trace * tr = &state->traces[i]; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, i); for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { @@ -1920,7 +1921,7 @@ static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data) static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { output_transfer_task_state_t *st = (output_transfer_task_state_t *) data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int start_chunk_idx = i * st->n_chunks_per_task; htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); @@ -1955,6 +1956,170 @@ typedef struct { uint32_t dma_step_rows_shift; } activation_transfer_task_state_t; +typedef struct { + __fp16 *dst; + const float *src; + uint32_t n_rows; + uint32_t k_block; + uint32_t k_stride; + uint32_t k_valid; + uint32_t n_col_chunks; + struct fastdiv_values n_threads_div; + float *vtcm_f32_act; + size_t vtcm_f32_act_bytes; + struct htp_thread_trace *traces; + struct htp_context *ctx; + uint32_t dma_step_rows; + uint32_t dma_step_rows_shift; +} activation_transfer_col_chunk_state_t; + +static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( + dma_queue *dma_q, + __fp16 *restrict vtcm_dst, + const float *restrict src, + uint32_t n_rows, + uint32_t k_block, + uint32_t k_stride, + uint32_t k_chunk_valid, + uint32_t c_first, + uint32_t c_len, + float *thread_f32_act, + struct htp_thread_trace *tr, + uint32_t dma_step_rows, + uint32_t dma_step_rows_shift) { + + const uint32_t R = dma_step_rows; + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + + const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift; + + // Push step 0 + if (n_steps > 0 && n_rows > 0) { + uint32_t nrows_to_fetch = hex_smin(n_rows, R); + dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src + c_first), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + // Push step 1 + if (n_steps > 1) { + uint32_t next_r = R * 1; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + const float *next_src = src + next_r * k_stride + c_first; + float *next_buf = thread_f32_act + 1 * R * c_len; + dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + } + for (uint32_t s = 0; s < n_steps; ++s) { + uint32_t r = s << dma_step_rows_shift; + float *curr_buf = thread_f32_act; + + if (r < n_rows) { + curr_buf = (float *) dma_queue_pop(dma_q).dst; + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + float *pair_buf = curr_buf + (p << 1) * c_len; + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); + + transfer_activation_row_pair_fp32_to_fp16_col_chunk( + vtcm_dst, pair_buf, pair_buf + c_len, row_idx, k_block, c_first, c_len, k_chunk_valid, r0_valid, r1_valid + ); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + + // Push step s + 2 + uint32_t next_s = s + 2; + uint32_t next_r = next_s << dma_step_rows_shift; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + const float *next_src = src + next_r * k_stride + c_first; + dma_queue_push(dma_q, dma_make_ptr(curr_buf, next_src), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + } +} + +static void transfer_activation_chunk_fp32_to_fp16_col_chunk( + __fp16 *restrict vtcm_dst, + const float *restrict src, + uint32_t n_rows, + uint32_t k_block, + uint32_t k_stride, + uint32_t c_first, + uint32_t c_len, + uint32_t k_chunk_valid) { + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + const uint32_t n_rows_tiled = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS; + + uint32_t r = 0; + + #pragma unroll(2) + for (r = 0; r < n_rows_tiled; r += 2) { + const float *ptr_in0 = src + (r + 0) * k_stride + c_first; + const float *ptr_in1 = src + (r + 1) * k_stride + c_first; + + transfer_activation_row_pair_fp32_to_fp16_col_chunk( + vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, true, true + ); + } + + for (; r < n_rows_padded; r += 2) { + const bool row0_valid = r < n_rows; + const bool row1_valid = (r + 1) < n_rows; + + const float *ptr_in0 = row0_valid ? (src + (r + 0) * k_stride + c_first) : NULL; + const float *ptr_in1 = row1_valid ? (src + (r + 1) * k_stride + c_first) : NULL; + + transfer_activation_row_pair_fp32_to_fp16_col_chunk( + vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, row0_valid, row1_valid + ); + } +} + +static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { + activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data; + struct htp_thread_trace * tr = &st->traces[i]; + + uint32_t n_blocks = st->k_block / 32; + uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t c_first = b_first * 32; + uint32_t c_last = b_last * 32; + uint32_t c_len = c_last - c_first; + + if (c_len == 0) { + return; + } + + uint32_t k_chunk_valid = 0; + if (st->k_valid > c_first) { + k_chunk_valid = hex_smin(st->k_valid, c_last) - c_first; + } + + __fp16 *dst = st->dst; + const float *src = st->src; + + if (st->vtcm_f32_act) { + size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128); + float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes); + + transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( + st->ctx->dma[i], dst, src, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, + c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift + ); + } else { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first); + transfer_activation_chunk_fp32_to_fp16_col_chunk( + dst, src, st->n_rows, st->k_block, st->k_stride, c_first, c_len, k_chunk_valid + ); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first); + } +} + static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( dma_queue *dma_q, __fp16 *restrict vtcm_dst, @@ -2024,7 +2189,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { int chunk_idx = task_id * st->n_chunks_per_task; @@ -2085,15 +2250,16 @@ typedef struct { static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; - int start_row = st->start_row + chunk_idx * chunk_size; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; int n_rows = hex_smin(st->cne1 - start_row, chunk_size); if (n_rows > 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); transfer_activation_chunk_fp32_to_fp16_gathered( - st->dst, st->src, start_row, n_rows, st->k_block, + st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block, st->matrix_rows, st->cur_a, st->mapping_stride, st->ne11, &st->ne11_div, st->nb11, st->nb12, st->cne1, st->k_valid); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); @@ -2102,15 +2268,16 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigne static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; - int start_row = st->start_row + chunk_idx * chunk_size; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; int n_rows = hex_smin(st->cne1 - start_row, chunk_size); if (n_rows > 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); transfer_activation_chunk_fp32_to_fp16_gathered_flat( - st->dst, st->src, start_row, n_rows, st->k_block, + st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block, st->matrix_rows, st->cur_a, st->mapping_stride, st->nb12, st->cne1, st->k_valid); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); @@ -2119,15 +2286,16 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, un static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) { output_transfer_scattered_task_state_t *st = data; - struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; + struct htp_thread_trace * tr = &st->traces[i]; int chunk_idx = i; int chunk_size = st->n_chunks_per_task; - int start_row = st->start_row + chunk_idx * chunk_size; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; int n_rows = hex_smin(st->cne1 - start_row, chunk_size); if (n_rows > 0) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx); transfer_output_chunk_fp16_to_fp32_scattered( - st->dst, st->vtcm_src, start_row, n_rows, st->n_cols, + st->dst, st->vtcm_src, start_row, vtcm_start_row, n_rows, st->n_cols, st->matrix_rows, st->cur_a, st->mapping_stride, st->dst_nb1, st->dst_nb2, st->cne1); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx); @@ -2210,42 +2378,81 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, } } -static void transfer_activation_chunk_threaded( - struct htp_context *ctx, - __fp16 *dst, - const float *src, - int n_rows, - int k_block, - int k_stride, - int n_threads, - int k_valid, - float *vtcm_f32_act, - size_t vtcm_f32_act_bytes) { +struct activation_transfer_params { + struct htp_context * ctx; + __fp16 * dst; + const float * src; + int n_rows; + int k_block; + int k_stride; + int n_threads; + const struct fastdiv_values * act_threads_div; + const struct fastdiv_values * k_div; + int k_valid; + float * vtcm_f32_act; + size_t vtcm_f32_act_bytes; +}; + +static void transfer_activation_chunk_threaded(const struct activation_transfer_params * params) { + struct htp_context * ctx = params->ctx; + __fp16 * dst = params->dst; + const float * src = params->src; + int n_rows = params->n_rows; + int k_block = params->k_block; + int k_stride = params->k_stride; + int n_threads = params->n_threads; + const struct fastdiv_values * act_threads_div = params->act_threads_div; + const struct fastdiv_values * k_div = params->k_div; + int k_valid = params->k_valid; + float * vtcm_f32_act = params->vtcm_f32_act; + size_t vtcm_f32_act_bytes = params->vtcm_f32_act_bytes; + if (n_rows <= 0) { return; } + const size_t n_tasks = (n_rows + 31) >> 5; + if (n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) { + // Calculate step rows parameters for column-chunked dma pipelining + uint32_t dma_step_rows = 2; + uint32_t dma_step_rows_shift = 1; + if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) { + size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float); + size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); + if (dma_step_rows_max >= 4) { + dma_step_rows = 4; + dma_step_rows_shift = 2; + } + } + + activation_transfer_col_chunk_state_t col_state; + col_state.dst = dst; + col_state.src = src; + col_state.n_rows = n_rows; + col_state.k_block = k_block; + col_state.k_stride = k_stride; + col_state.k_valid = k_valid; + col_state.n_col_chunks = n_threads; + col_state.n_threads_div = *act_threads_div; + col_state.vtcm_f32_act = vtcm_f32_act; + col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes; + col_state.traces = ctx->trace; + col_state.ctx = ctx; + col_state.dma_step_rows = dma_step_rows; + col_state.dma_step_rows_shift = dma_step_rows_shift; + + worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads); + return; + } + assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0); size_t n_tot_chunks = n_rows; size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address - uint32_t dma_step_rows = 2; - uint32_t dma_step_rows_shift = 1; - if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) { - size_t thread_scratch_elements = vtcm_f32_act_bytes / (n_threads * sizeof(float)); - size_t dma_step_rows_max = (thread_scratch_elements / 2) / k_block; - if (dma_step_rows_max >= 4) { - dma_step_rows = 4; - dma_step_rows_shift = 2; - } else { - dma_step_rows = 2; - dma_step_rows_shift = 1; - } - } - activation_transfer_task_state_t state; - state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task; + state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; @@ -2258,7 +2465,18 @@ static void transfer_activation_chunk_threaded( state.vtcm_f32_act = vtcm_f32_act; int active_threads = hex_smin(n_threads, (int)state.n_tasks); - state.vtcm_f32_act_bytes_per_thread = (vtcm_f32_act_bytes / active_threads) & ~127u; + state.vtcm_f32_act_bytes_per_thread = hex_align_down(vtcm_f32_act_bytes / active_threads, 128); + + uint32_t dma_step_rows = 2; + uint32_t dma_step_rows_shift = 1; + if (vtcm_f32_act && state.vtcm_f32_act_bytes_per_thread > 0 && k_block > 0) { + size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float); + size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); + if (dma_step_rows_max >= 4) { + dma_step_rows = 4; + dma_step_rows_shift = 2; + } + } state.dma_step_rows = dma_step_rows; state.dma_step_rows_shift = dma_step_rows_shift; @@ -2321,9 +2539,14 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int pipeline, int n_threads, int act_threads, + const struct fastdiv_values * act_threads_div, + const struct fastdiv_values * k_div, int tile_size, int aligned_tile_size, int vtcm_size) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + if (k % 32 != 0 || n % 32 != 0) { return -1; } if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN)) { return -1; } @@ -2393,6 +2616,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + if (pipeline) { // --- Asynchronous Pipelined Loop --- hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors @@ -2403,7 +2628,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; - transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes); + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .src = activation + mr * act_stride, + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + }; + transfer_activation_chunk_threaded(&act_params); // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1) const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); @@ -2480,7 +2719,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); - transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes); + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .src = activation + mr * act_stride, + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + }; + transfer_activation_chunk_threaded(&act_params); // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { @@ -2570,7 +2823,8 @@ static inline const float *hmx_mm_src2_batch_ptr(const hmx_mm_f16_f32_batched_pa static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) { + int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size, + const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) { int ret = 0; for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { @@ -2582,14 +2836,17 @@ static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, params->act_stride, params->weight_stride * (int)sizeof(__fp16), HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n, m_chunk, n_chunk, pipeline, n_threads, act_threads, - 0, 0, vtcm_size); + act_threads_div, k_div, 0, 0, vtcm_size); } } return ret; } static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, - int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) { + int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, + const struct fastdiv_values * act_threads_div, + const struct fastdiv_values * k_div, + int vtcm_size) { if (params->act_stride < params->k || params->weight_stride < params->k || params->dst_stride < params->n) { return -1; } if (params->ne02 <= 0 || params->ne03 <= 0 || params->ne12 <= 0 || params->ne13 <= 0) { return -1; } if (params->ne12 % params->ne02 != 0 || params->ne13 % params->ne03 != 0) { return -1; } @@ -2604,9 +2861,12 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ // Grouped path is only valid if group_size > 1 and it fits within VTCM budget. bool run_grouped = (group_size > 1 && (size_t)vtcm_size <= vtcm_budget); if (!run_grouped) { - return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size); + return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); } + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const size_t vec_dot_size = params->k * sizeof(__fp16); const bool use_dma_activation = (params->act_stride > params->k); @@ -2622,7 +2882,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ if (L.total_bytes > vtcm_budget) { FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); - return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); } uint8_t * const base = (uint8_t *) ctx->vtcm_base; @@ -2644,6 +2905,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + hmx_matmul_job_t job; for (int b3 = 0; b3 < params->ne13; ++b3) { @@ -2662,9 +2925,21 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ for (int g = 0; g < group_size; ++g) { const float *activation_chunk = hmx_mm_activation_batch_ptr(params, b2_base + g, b3) + mr * params->act_stride; __fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride; - transfer_activation_chunk_threaded(ctx, vtcm_act_g, - activation_chunk, (int) n_rows, - params->k, params->act_stride, act_threads, params->k, vtcm_f32_act, L.act_f32_bytes); + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_act_g, + .src = activation_chunk, + .n_rows = (int) n_rows, + .k_block = params->k, + .k_stride = params->act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = params->k, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + }; + transfer_activation_chunk_threaded(&act_params); } // Prologue: Push A0 and A1 (if exists) @@ -2835,6 +3110,9 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const struct mmid_row_mapping *matrix_rows, int cur_a, int mapping_stride) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const int cne1 = m; const int m_padded = hex_align_up(m, 32); @@ -2913,6 +3191,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + hmx_matmul_job_t job; for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) { @@ -2980,10 +3260,6 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k const int act_stride = (int)(src1->nb[1] / sizeof(float)); const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16)); - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } - const float * src2_ptr = NULL; uint32_t src2_stride = 0; size_t src2_nb2 = 0; @@ -3027,6 +3303,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, + &kparams->div_n_act_threads, + &kparams->div_ne00_padded, kparams->vtcm_size); } else { ret = hmx_mm_2d_f32( @@ -3035,6 +3313,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k (int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0], kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, + &kparams->div_n_act_threads, + &kparams->div_ne00_padded, kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size ); } @@ -3103,6 +3383,10 @@ static int hvx_mm_matmul_id( bool must_free_mapping ) { htp_matmul_tensors_preamble; + + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; const struct htp_tensor * restrict ids = octx->src[2]; const size_t src0_row_size = nb01; @@ -3175,8 +3459,9 @@ static int hvx_mm_matmul_id( mmctx->n_quant_tasks = n_quant_tasks; atomic_init(&mmctx->quant_barrier, n_quant_tasks); - const uint32_t n_matmul_jobs = octx->n_threads; - worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, n_matmul_jobs); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, octx->n_threads); if (must_free_mapping) free(mapping_buf); return HTP_STATUS_OK; @@ -3185,6 +3470,9 @@ static int hvx_mm_matmul_id( int op_matmul_id(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + struct htp_mm_context mmctx_struct = {0}; struct htp_mm_context * mmctx = &mmctx_struct; mmctx->octx = octx; @@ -3262,10 +3550,7 @@ int op_matmul_id(struct htp_ops_context * octx) { } } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - if (must_free_mapping) free(mapping_buf); - return HTP_STATUS_OK; - } + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); if (kparams->n_hmx) { return hmx_mm_op_matmul_id(octx, mmctx, matrix_row_counts, matrix_rows, mapping_buf, must_free_mapping); @@ -3275,6 +3560,9 @@ int op_matmul_id(struct htp_ops_context * octx) { } int op_matmul_qkv(struct htp_ops_context * octx) { + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const struct htp_tensor * restrict src0 = octx->src[0]; // Wk const struct htp_tensor * restrict src1 = octx->src[1]; // x const struct htp_tensor * restrict src2 = octx->src[2]; // Wv @@ -3379,9 +3667,6 @@ int op_matmul_qkv(struct htp_ops_context * octx) { mmctx->vtcm_src3_size_per_thread = L.src3_bytes / octx->n_threads; mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; - mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; mmctx->n_quant_tasks = n_quant_tasks; @@ -3413,12 +3698,18 @@ int op_matmul_qkv(struct htp_ops_context * octx) { } else { matmul_job_func = hvx_mm_qkv_2d; } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); return HTP_STATUS_OK; } int op_matmul_ffn(struct htp_ops_context * octx) { + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + const struct htp_tensor * restrict src0 = octx->src[0]; // Wgate const struct htp_tensor * restrict src1 = octx->src[1]; // y const struct htp_tensor * restrict src2 = octx->src[2]; // Wup @@ -3516,9 +3807,6 @@ int op_matmul_ffn(struct htp_ops_context * octx) { mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads; mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; - mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; mmctx->quant_task_func = quant_task_func; mmctx->n_quant_tasks = n_quant_tasks; @@ -3550,6 +3838,9 @@ int op_matmul_ffn(struct htp_ops_context * octx) { } else { matmul_job_func = hvx_mm_ffn_2d; } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 2e131bc3d0..4450446609 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -95,6 +95,8 @@ struct htp_mm_kernel_params { struct fastdiv_values div_r2; struct fastdiv_values div_r3; struct fastdiv_values div_ne11; + struct fastdiv_values div_n_act_threads; + struct fastdiv_values div_ne00_padded; }; #if defined(__cplusplus) @@ -643,6 +645,136 @@ static inline size_t htp_mm_hmx_get_batched_vtcm_size( return L.total_bytes; } +static inline bool htp_mm_hmx_solve_batched_params( + int wtype, + uint32_t k, + uint32_t ne01_padded, + uint32_t ne11, + uint32_t group_size, + bool use_dma_activation, + int n_threads, + bool pipeline, + size_t vtcm_budget, + size_t * m_chunk_out, + size_t * n_chunk_out, + int * act_threads_out, + size_t * vtcm_size_out +) { + size_t best_mblocks = SIZE_MAX; + int best_act_threads = 0; + size_t best_m_chunk = 0; + size_t best_n_chunk = 0; + size_t best_vtcm_size = 0; + + int act_threads = n_threads; + while (act_threads >= 1) { + size_t group_overhead = 256; + size_t group_size_per_n, group_size_per_m, group_size_per_mn; + htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); + + size_t m_chunk_candidate = 0; + size_t n_chunk_candidate = 0; + size_t vtcm_size_candidate = 0; + + if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, + (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, + &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { + size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads); + if (exact_size <= vtcm_budget) { + size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + best_mblocks = mblocks; + best_act_threads = act_threads; + best_m_chunk = m_chunk_candidate; + best_n_chunk = n_chunk_candidate; + best_vtcm_size = exact_size; + } + } + } + if (act_threads == 1) { + act_threads = 0; + } else { + act_threads /= 2; + } + } + + if (best_act_threads > 0) { + *m_chunk_out = best_m_chunk; + *n_chunk_out = best_n_chunk; + *vtcm_size_out = best_vtcm_size; + *act_threads_out = best_act_threads; + return true; + } + return false; +} + +static inline bool htp_mm_hmx_solve_2d_params( + int wtype, + uint32_t k, + uint32_t m_id_rows, + uint32_t ne01_padded, + uint32_t ne11_padded, + uint32_t m_for_cost, + int n_threads, + bool pipeline, + bool is_matmul_id, + uint32_t aligned_tile_size, + size_t vtcm_budget, + size_t * m_chunk_out, + size_t * n_chunk_out, + int * act_threads_out, + size_t * vtcm_size_out +) { + size_t best_mblocks = SIZE_MAX; + int best_act_threads = 0; + size_t best_m_chunk = 0; + size_t best_n_chunk = 0; + size_t best_vtcm_size = 0; + + const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; + + int act_threads = n_threads; + while (act_threads >= 1) { + size_t simple_2d_overhead = 256; + size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; + htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); + + size_t m_chunk_candidate = 0; + size_t n_chunk_candidate = 0; + size_t vtcm_size_candidate = 0; + + if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, + (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, + &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size); + if (exact_size <= vtcm_budget) { + size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + best_mblocks = mblocks; + best_act_threads = act_threads; + best_m_chunk = m_chunk_candidate; + best_n_chunk = n_chunk_candidate; + best_vtcm_size = exact_size; + } + } + } + if (act_threads == 1) { + act_threads = 0; + } else { + act_threads /= 2; + } + } + + if (best_act_threads > 0) { + *m_chunk_out = best_m_chunk; + *n_chunk_out = best_n_chunk; + *vtcm_size_out = best_vtcm_size; + *act_threads_out = best_act_threads; + return true; + } + return false; +} + #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index d16dc7d38e..5bc7d74f5e 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -18,6 +18,7 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "htp-ops.h" +#include "htp-tensor.h" // Redefined the rope type constants as we can't include ggml.h #define HTP_ROPE_TYPE_NORMAL 0 @@ -712,17 +713,11 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { } int op_rope(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - switch (octx->src[0]->type) { case HTP_TYPE_F32: - err = execute_op_rope_f32(octx); - break; + return execute_op_rope_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index a71107f104..82792c610b 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -19,6 +19,7 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "htp-vtcm.h" #include "hex-profile.h" @@ -397,7 +398,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat struct htp_ops_context * octx = uctx->octx; \ const struct htp_tensor * src = octx->src[0]; \ const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ htp_unary_preamble; \ \ @@ -558,7 +559,7 @@ static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void struct htp_ops_context * octx = uctx->octx; \ const struct htp_tensor * src = octx->src[0]; \ const struct htp_tensor * dst = octx->dst; \ - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ \ htp_unary_preamble; \ \ @@ -922,17 +923,11 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { } int op_unary(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - switch (octx->src[0]->type) { case HTP_TYPE_F32: - err = execute_op_unary_f32(octx); - break; + return execute_op_unary_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/work-queue.c b/ggml/src/ggml-hexagon/htp/work-queue.c new file mode 100644 index 0000000000..bb73e205a4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/work-queue.c @@ -0,0 +1,244 @@ +#include "work-queue.h" +#include "hex-utils.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "HAP_farf.h" + +#define LOWEST_USABLE_QURT_PRIO (254) + +// internal structure kept in thread-local storage per instance of work queue +typedef struct { + work_queue_t queue; + unsigned int id; +} worker_context_t; + +struct work_queue_task_s { + work_queue_func_t func; + void * data; + unsigned int n_threads; + atomic_uint barrier; +}; + +// internal structure kept in thread-local storage per instance of work queue +struct work_queue_s { + atomic_uint seqn; // seqno used to detect new jobs + atomic_uint idx_read; // Updated by producer (pop/reclaim) + unsigned int idx_write; // Updated by producer (push) + uint32_t idx_mask; + uint32_t capacity; + + qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers + worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts + void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers + unsigned int n_threads; // total threads (workers + main) + unsigned int n_workers; // number of active threads (just workers) + + atomic_bool active; // workers are polling/active + atomic_bool killed; // threads need to exit + bool external_mem; // memory owned externally + + struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE))); +}; + +static void work_queue_thread(void * context) { + worker_context_t * me = (worker_context_t *) context; + work_queue_t q = me->queue; + + FARF(HIGH, "work-queue: thread %u started", me->id); + + unsigned int prev_seqn = 0; + + while (!atomic_load_explicit(&q->killed, memory_order_relaxed)) { + unsigned int seqn = atomic_load_explicit(&q->seqn, memory_order_acquire); + if (seqn == prev_seqn) { + if (atomic_load_explicit(&q->active, memory_order_relaxed)) { + hex_pause(); + } else { + qurt_futex_wait(&q->seqn, prev_seqn); + } + continue; + } + + prev_seqn = seqn; + + // Process all active tasks in the queue + unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed); + unsigned int iw = q->idx_write; + + while (ir != iw) { + struct work_queue_task_s * task = &q->queue[ir]; + + unsigned int n = task->n_threads; + unsigned int i = me->id; + if (i < n) { + task->func(n, i, task->data); + + atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); + } else { + while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) { + hex_pause(); + } + } + + ir = (ir + 1) & q->idx_mask; + } + } + + FARF(HIGH, "work-queue: thread %u stopped", me->id); +} + +bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n > q->n_threads) { + FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads); + return false; + } + + unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed); + unsigned int iw = q->idx_write; + + if (((iw + 1) & q->idx_mask) == ir) { + FARF(ERROR, "work-queue-push: queue is full\n"); + return false; + } + + struct work_queue_task_s * task = &q->queue[iw]; + task->func = func; + task->data = data; + task->n_threads = n; + atomic_store_explicit(&task->barrier, n, memory_order_relaxed); + + q->idx_write = (iw + 1) & q->idx_mask; + + // publish job to workers (already awake and polling) + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + + // main thread runs job #0 + func(n, 0, data); + + atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); + + while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) { + hex_pause(); + } + + atomic_thread_fence(memory_order_acquire); + + atomic_store_explicit(&q->idx_read, (ir + 1) & q->idx_mask, memory_order_relaxed); + + return true; +} + +size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { + capacity = hex_ceil_pow2(capacity); + uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; + size_t size_stacks = stack_size * n_workers; + size_t size_q = hex_align_up(sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s), HEX_L2_LINE_SIZE); + return size_stacks + size_q; +} + +size_t work_queue_alignof(void) { + return 4096; +} + +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { + capacity = hex_ceil_pow2(capacity); + uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; + unsigned char * mem_blob = (unsigned char *) ptr; + + work_queue_t q = (work_queue_t) (mem_blob + stack_size * n_workers); + memset(q, 0, sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s)); + + q->n_threads = n_threads; + q->n_workers = n_workers; + q->external_mem = true; + q->capacity = capacity; + + for (unsigned int i = 0; i < n_workers; i++) { + q->stack[i] = mem_blob; mem_blob += stack_size; + q->thread[i] = 0; + q->context[i].id = i + 1; + q->context[i].queue = q; + } + + atomic_init(&q->idx_read, 0); + atomic_init(&q->seqn, 0); + atomic_init(&q->active, false); + q->idx_write = 0; + q->idx_mask = capacity - 1; + q->killed = 0; + for (int i = 0; i < (int) capacity; i++) { + atomic_init(&q->queue[i].barrier, 0); + q->queue[i].func = NULL; + q->queue[i].data = NULL; + q->queue[i].n_threads = 0; + } + + // launch the workers + qurt_thread_attr_t attr; + qurt_thread_attr_init(&attr); + + for (unsigned int i = 0; i < n_workers; i++) { + qurt_thread_attr_set_stack_addr(&attr, q->stack[i]); + qurt_thread_attr_set_stack_size(&attr, stack_size); + + char thread_name[32]; + snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i); + qurt_thread_attr_set_name(&attr, thread_name); + + // set up priority - by default, match the creating thread's prio + int prio = qurt_thread_get_priority(qurt_thread_get_id()); + if (prio < 1) { + prio = 1; + } + if (prio > LOWEST_USABLE_QURT_PRIO) { + prio = LOWEST_USABLE_QURT_PRIO; + } + + qurt_thread_attr_set_priority(&attr, prio); + + int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]); + if (err) { + FARF(ERROR, "Could not launch worker threads!"); + work_queue_free(q); + return NULL; + } + } + + return q; +} + +void work_queue_free(work_queue_t q) { + if (!q) { return; } + + atomic_store_explicit(&q->killed, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + qurt_futex_wake(&q->seqn, q->n_workers); + + for (unsigned int i = 0; i < q->n_workers; i++) { + if (q->thread[i]) { + int status; + (void) qurt_thread_join(q->thread[i], &status); + } + } +} + +void work_queue_wakeup(work_queue_t q) { + if (!atomic_load_explicit(&q->active, memory_order_relaxed)) { + atomic_store_explicit(&q->active, true, memory_order_release); + // Increment seqn and wake workers to transition them out of sleep + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + qurt_futex_wake(&q->seqn, q->n_workers); + } +} + +void work_queue_suspend(work_queue_t q) { + atomic_store_explicit(&q->active, false, memory_order_release); +} diff --git a/ggml/src/ggml-hexagon/htp/work-queue.h b/ggml/src/ggml-hexagon/htp/work-queue.h new file mode 100644 index 0000000000..09ca4b1f43 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/work-queue.h @@ -0,0 +1,38 @@ +#ifndef HTP_WORK_QUEUE_H +#define HTP_WORK_QUEUE_H + +#include +#include +#include + +typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *); + +struct work_queue_s; +typedef struct work_queue_s * work_queue_t; + +#define WORK_QUEUE_MAX_N_THREADS 10 + +size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +size_t work_queue_alignof(void); +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +void work_queue_free(work_queue_t q); + +void work_queue_wakeup(work_queue_t q); +void work_queue_suspend(work_queue_t q); + +bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); + +static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n <= 1) { + func(n, 0, data); + return true; + } + return work_queue_run_async(q, func, data, n); +} + +// Legacy compatibility +typedef work_queue_func_t worker_callback_t; +#define worker_pool_run_func work_queue_run +#define worker_pool work_queue + +#endif // #ifndef HTP_WORK_QUEUE_H diff --git a/ggml/src/ggml-hexagon/htp/worker-pool.c b/ggml/src/ggml-hexagon/htp/worker-pool.c deleted file mode 100644 index 50960d2c75..0000000000 --- a/ggml/src/ggml-hexagon/htp/worker-pool.c +++ /dev/null @@ -1,305 +0,0 @@ -#include "worker-pool.h" -#include "hex-utils.h" - -#include -#include - -#include -#include -#include -#include -#include - -#include "HAP_farf.h" - -#define LOWEST_USABLE_QURT_PRIO (254) - -struct worker_pool_s; - -// internal structure kept in thread-local storage per instance of worker pool -typedef struct { - struct worker_pool_s * pool; - unsigned int id; -} worker_context_t; - -// internal structure kept in thread-local storage per instance of worker pool -typedef struct worker_pool_s { - worker_pool_job_t job[MAX_NUM_WORKERS]; // list of job descriptors - qurt_thread_t thread[MAX_NUM_WORKERS]; // thread ID's of the workers - worker_context_t context[MAX_NUM_WORKERS]; // worker contexts - void * stack[MAX_NUM_WORKERS]; // thread stack pointers - unsigned int n_threads; // number of workers in this pool - - atomic_uint seqn; // seqno used to detect new jobs - atomic_uint next_job; // next job index - atomic_uint n_pending; // number of pending jobs - atomic_uint n_jobs; // number of current jobs - atomic_bool killed; // threads need to exit -} worker_pool_t; - -static void worker_pool_main(void * context) { - worker_context_t * me = (worker_context_t *) context; - worker_pool_t * pool = me->pool; - - FARF(HIGH, "worker-pool: thread %u started", me->id); - - unsigned int prev_seqn = 0; - unsigned int poll_cnt = WORKER_POOL_POLL_COUNT; - while (!atomic_load(&pool->killed)) { - unsigned int seqn = atomic_load(&pool->seqn); - if (seqn == prev_seqn) { - // drop HVX context while spinning - if (poll_cnt > 1 && poll_cnt == WORKER_POOL_POLL_COUNT) { - qurt_hvx_unlock(); - } - if (--poll_cnt) { - hex_pause(); - continue; - } - qurt_futex_wait(&pool->seqn, prev_seqn); - poll_cnt = WORKER_POOL_POLL_COUNT; - continue; - } - - prev_seqn = seqn; - poll_cnt = WORKER_POOL_POLL_COUNT; - - // New job - unsigned int n = atomic_load(&pool->n_jobs); - unsigned int i = atomic_fetch_add(&pool->next_job, 1); - if (i >= n) { - // Spurious wakeup - continue; - } - - pool->job[i].func(n, i, pool->job[i].data); - - atomic_fetch_sub(&pool->n_pending, 1); - } - - FARF(HIGH, "worker-pool: thread %u stopped", me->id); -} - -AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context, uint32_t n_threads, uint32_t stack_size) { - int err = 0; - - if (NULL == context) { - FARF(ERROR, "NULL context passed to worker_pool_init()."); - return AEE_EBADPARM; - } - - // Allocations - int size = (stack_size * n_threads) + (sizeof(worker_pool_t)); - - unsigned char * mem_blob = (unsigned char *) malloc(size); - if (!mem_blob) { - FARF(ERROR, "Could not allocate memory for worker pool!!"); - return AEE_ENOMEMORY; - } - - worker_pool_t * me = (worker_pool_t *) (mem_blob + stack_size * n_threads); - - // name for the first worker, useful in debugging threads - char name[19]; - snprintf(name, 12, "0x%8x:", (int) me); - strcat(name, "worker0"); - me->n_threads = n_threads; - - // initializations - for (unsigned int i = 0; i < me->n_threads; i++) { - me->stack[i] = NULL; - me->thread[i] = 0; - - me->context[i].id = i; - me->context[i].pool = me; - } - - // initialize job queue - me->n_pending = 0; - me->n_jobs = 0; - me->next_job = 0; - me->seqn = 0; - me->killed = 0; - - // launch the workers - qurt_thread_attr_t attr; - qurt_thread_attr_init(&attr); - - for (unsigned int i = 0; i < me->n_threads; i++) { - // set up stack - me->stack[i] = mem_blob; - mem_blob += stack_size; - qurt_thread_attr_set_stack_addr(&attr, me->stack[i]); - qurt_thread_attr_set_stack_size(&attr, stack_size); - - // set up name - qurt_thread_attr_set_name(&attr, name); - name[17] = (name[17] + 1); - // name threads context:worker0, context:worker1, .. (recycle at 9, but num threads should be less than that anyway) - if (name[17] > '9') { - name[17] = '0'; - } - - // set up priority - by default, match the creating thread's prio - int prio = qurt_thread_get_priority(qurt_thread_get_id()); - - if (prio < 1) { - prio = 1; - } - if (prio > LOWEST_USABLE_QURT_PRIO) { - prio = LOWEST_USABLE_QURT_PRIO; - } - - qurt_thread_attr_set_priority(&attr, prio); - - // launch - err = qurt_thread_create(&me->thread[i], &attr, worker_pool_main, (void *) &me->context[i]); - if (err) { - FARF(ERROR, "Could not launch worker threads!"); - worker_pool_release((worker_pool_context_t *) &me); - return AEE_EQURTTHREADCREATE; - } - } - *context = (worker_pool_context_t *) me; - return AEE_SUCCESS; -} - -AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads) { - return worker_pool_init_with_stack_size(context, n_threads, WORKER_THREAD_STACK_SZ); -} - -// clean up worker pool -void worker_pool_release(worker_pool_context_t * context) { - worker_pool_t * me = (worker_pool_t *) *context; - - // if no worker pool exists, return error. - if (NULL == me) { - return; - } - - atomic_store(&me->killed, 1); - atomic_fetch_add(&me->seqn, 1); - qurt_futex_wake(&me->seqn, me->n_threads); - - // de-initializations - for (unsigned int i = 0; i < me->n_threads; i++) { - if (me->thread[i]) { - int status; - (void) qurt_thread_join(me->thread[i], &status); - } - } - - // free allocated memory (were allocated as a single buffer starting at stack[0]) - if (me->stack[0]) { - free(me->stack[0]); - } - - *context = NULL; -} - -// run jobs -AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n) { - worker_pool_t * me = (worker_pool_t *) context; - if (NULL == me) { - FARF(ERROR, "worker-pool: invalid context"); - return AEE_EBADPARM; - } - - if (n > me->n_threads) { - FARF(ERROR, "worker-pool: invalid number of jobs %u for n-threads %u", n, me->n_threads); - return AEE_EBADPARM; - } - - memcpy(me->job, job, sizeof(worker_pool_job_t) * n); - - if (n > 1) { - atomic_store(&me->next_job, 1); - atomic_store(&me->n_jobs, n); - atomic_store(&me->n_pending, n - 1); - - // wake up workers - atomic_fetch_add(&me->seqn, 1); - qurt_futex_wake(&me->seqn, n - 1); - } - - // main thread runs job #0 - me->job[0].func(n, 0, me->job[0].data); - - if (n > 1) { - while (atomic_load(&me->n_pending)) - ; - } - - return 0; -} - -// run func -AEEResult worker_pool_run_func(worker_pool_context_t context, worker_callback_t func, void * data, unsigned int n) { - worker_pool_job_t job[n]; - - for (unsigned int i = 0; i < n; i++) { - job[i].func = func; - job[i].data = data; - } - - return worker_pool_run_jobs(context, job, n); -} - -AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio) { - worker_pool_t * me = (worker_pool_t *) context; - - // if no worker pool exists, return error. - if (!me) { - return AEE_ENOMORE; - } - - int result = AEE_SUCCESS; - if (prio < 1) { - prio = 1; - } - if (prio > LOWEST_USABLE_QURT_PRIO) { - prio = LOWEST_USABLE_QURT_PRIO; - } - - for (unsigned int i = 0; i < me->n_threads; i++) { - int res = qurt_thread_set_priority(me->thread[i], (unsigned short) prio); - if (0 != res) { - result = AEE_EBADPARM; - FARF(ERROR, "QURT failed to set priority of thread %d, ERROR = %d", me->thread[i], res); - } - } - - return result; -} - -AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids) { - worker_pool_t * me = (worker_pool_t *) context; - if (!me) { - FARF(ERROR, "worker-pool: invalid context"); - return AEE_EBADPARM; - ; - } - - for (int i = 0; i < me->n_threads; i++) { - tids[i] = me->thread[i]; - } - - return AEE_SUCCESS; -} - -AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio) { - worker_pool_t * me = (worker_pool_t *) context; - if (!me) { - FARF(ERROR, "worker-pool: invalid context"); - return AEE_EBADPARM; - } - - int priority = qurt_thread_get_priority(me->thread[0]); - if (priority > 0) { - *prio = priority; - return 0; - } else { - *prio = 0; - return AEE_EBADSTATE; - } -} diff --git a/ggml/src/ggml-hexagon/htp/worker-pool.h b/ggml/src/ggml-hexagon/htp/worker-pool.h deleted file mode 100644 index cba692126a..0000000000 --- a/ggml/src/ggml-hexagon/htp/worker-pool.h +++ /dev/null @@ -1,65 +0,0 @@ -#ifndef HTP_WORKER_POOL_H -#define HTP_WORKER_POOL_H - -// MACRO enables function to be visible in shared-library case. -#define WORKERPOOL_API __attribute__((visibility("default"))) - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/// signature of callbacks to be invoked by worker threads -typedef void (*worker_callback_t)(unsigned int n, unsigned int i, void *); - -/// Typedef of worker_pool context -typedef void * worker_pool_context_t; - -/// descriptor for requested callback -typedef struct { - worker_callback_t func; - void * data; -} worker_pool_job_t; - -#define WORKER_THREAD_STACK_SZ (2 * 16384) - -/// Maximum supported number of worker threads. -#define MAX_NUM_WORKERS 10 - -#if __HVX_ARCH__ > 79 -#define WORKER_POOL_POLL_COUNT 2000 -#else -#define WORKER_POOL_POLL_COUNT 1 -#endif - -// Initialize worker pool. -WORKERPOOL_API AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads); - -// Initialize worker pool with custom stack size -WORKERPOOL_API AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context, - uint32_t n_threads, - uint32_t stack_size); - -// Kill worker threads and release worker pool resources -WORKERPOOL_API void worker_pool_release(worker_pool_context_t * context); - -// Run jobs with the worker pool. -WORKERPOOL_API AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n); - -WORKERPOOL_API AEEResult worker_pool_run_func(worker_pool_context_t context, - worker_callback_t func, - void * data, - unsigned int n); - -WORKERPOOL_API AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio); -WORKERPOOL_API AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio); -WORKERPOOL_API AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids); - -#ifdef __cplusplus -} -#endif - -#endif // #ifndef HTP_WORKER_POOL_H diff --git a/scripts/snapdragon/ggml-hexagon-profile.py b/scripts/snapdragon/ggml-hexagon-profile.py index 0f9240ddc6..2e096aaa93 100755 --- a/scripts/snapdragon/ggml-hexagon-profile.py +++ b/scripts/snapdragon/ggml-hexagon-profile.py @@ -458,15 +458,11 @@ def main(): if args.timeline: logger.info(f"\n# ASCII Timing {args.timeline.capitalize()}\n") - printed_cnt = 0 for op in ops: if args.timeline == "summary": print_ascii_summary(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'], op.get('evt_val')) elif args.timeline == "diagram": print_ascii_timeline(op['name'], op['dims'], op['types'], op['usec'], op['cycles'], op['trace_events'], op.get('evt_val')) - printed_cnt += 1 - if printed_cnt >= args.top: - break else: generate_report(ops, args.top, overrides, args.sort, pmu_name=final_pmu_name)