From 2e1fd76490af2bafd20ee7f2a27bd9f49774f674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Thu, 16 Jul 2026 15:23:23 +0200 Subject: [PATCH] TP: fix Phi3, Bert, Plamo2/3, ChatGLM (#25536) --- src/llama-model.cpp | 21 ++++++++++++++++----- tests/test-llama-archs.cpp | 4 +++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index eaf3f35d2d..4c10e4126f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -360,8 +360,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight"); static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight"); - static const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight"); - static const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias"); + static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight"); + static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias"); + static const std::regex pattern_ffn_gate_weight ("blk\\.\\d*\\.ffn_gate(_exps)?.weight"); + static const std::regex pattern_ffn_gate_bias ("blk\\.\\d*\\.ffn_gate(_exps)?.bias"); static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight"); static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight"); static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias"); @@ -469,10 +471,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str } // FFN - if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight)) { + if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_gate_weight)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight"); } - if (std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) { + if (std::regex_match(tensor_name, pattern_ffn_up_bias) || std::regex_match(tensor_name, pattern_ffn_gate_bias)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight"); } if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { @@ -556,6 +558,14 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa); return {{n_embd, 1}, {n_embd_gqa, 2}}; } + if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias)) { + const int64_t n_ff = hparams.n_ff(il); + // some models such as Phi 3 have fused up + gate tensors named "up" tensors, which need to be segmented + if (tensor->ne[axis] == 2*n_ff) { + return {{n_ff, 2}}; + } + return {{tensor->ne[axis], 1}}; + } if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) { const int64_t n_ff_exp = hparams.n_ff_exp; GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp); @@ -632,7 +642,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str } // FFN - if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || std::regex_match(tensor_name, pattern_ffn_up_gate_bias) || + if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) || + std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) || std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) { const int64_t blck_size_perf = std::lcm(blck_size, 128); GGML_ASSERT(segments.size() == 1); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 796d490749..57d33a627d 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -40,8 +40,10 @@ static double nmse(const std::vector & a, const std::vector & b) { } static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { + size_t seed = *(const size_t *) userdata; std::hash hasher; - std::mt19937 gen(hasher(tensor->name) + *(const size_t *) userdata); + seed ^= hasher(tensor->name); + std::mt19937 gen(seed); std::normal_distribution dis(0.0f, 1.0e-2f); const int64_t ne = ggml_nelements(tensor);