furyhawk/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-21 02:05:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

TP: fix Phi3, Bert, Plamo2/3, ChatGLM (#25536)

Browse Source
This commit is contained in:
Johannes Gäßler
2026-07-16 16:23:23 +03:00
committed by GitHub
parent 86b719bf21
commit 2e1fd76490
2 changed files with 19 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+16 -5
View File
@@ -360,8 +360,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");
static const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
static const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight");
static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias");
static const std::regex pattern_ffn_gate_weight ("blk\\.\\d*\\.ffn_gate(_exps)?.weight");
static const std::regex pattern_ffn_gate_bias ("blk\\.\\d*\\.ffn_gate(_exps)?.bias");
static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
@@ -469,10 +471,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
}
// FFN
if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight)) {
if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_gate_weight)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");
}
if (std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) {
if (std::regex_match(tensor_name, pattern_ffn_up_bias) || std::regex_match(tensor_name, pattern_ffn_gate_bias)) {
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");
}
if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
@@ -556,6 +558,14 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);
return {{n_embd, 1}, {n_embd_gqa, 2}};
}
if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias)) {
const int64_t n_ff = hparams.n_ff(il);
// some models such as Phi 3 have fused up + gate tensors named "up" tensors, which need to be segmented
if (tensor->ne[axis] == 2*n_ff) {
return {{n_ff, 2}};
}
return {{tensor->ne[axis], 1}};
}
if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {
const int64_t n_ff_exp = hparams.n_ff_exp;
GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);
@@ -632,7 +642,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
}
// FFN
if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || std::regex_match(tensor_name, pattern_ffn_up_gate_bias) ||
if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) ||
std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) ||
std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) {
const int64_t blck_size_perf = std::lcm(blck_size, 128);
GGML_ASSERT(segments.size() == 1);
tests/test-llama-archs.cpp
+3 -1
View File
@@ -40,8 +40,10 @@ static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
}
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
size_t seed = *(const size_t *) userdata;
std::hash<std::string> hasher;
std::mt19937 gen(hasher(tensor->name) + *(const size_t *) userdata);
seed ^= hasher(tensor->name);
std::mt19937 gen(seed);
std::normal_distribution<float> dis(0.0f, 1.0e-2f);
const int64_t ne = ggml_nelements(tensor);