diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 3734a67d6..621b5adf5 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -396,6 +396,12 @@ int main(int argc, char ** argv) { } else { usage(argv[0]); } + } else if (strcmp(argv[arg_idx], "--per-layer-token-embedding-type") == 0) { + if (arg_idx < argc-1) { + params.per_layer_token_embedding_type = parse_ggml_type(argv[++arg_idx]); + } else { + usage(argv[0]); + } } else if (strcmp(argv[arg_idx], "--ffn-gate-inp-type") == 0) { if (arg_idx < argc-1) { params.ffn_gate_inp_type = parse_ggml_type(argv[++arg_idx]); diff --git a/ggml/src/iqk/iqk_quantize.cpp b/ggml/src/iqk/iqk_quantize.cpp index b049aa584..35b21c2e2 100644 --- a/ggml/src/iqk/iqk_quantize.cpp +++ b/ggml/src/iqk/iqk_quantize.cpp @@ -8353,10 +8353,7 @@ const Modify * get_modify_info(ggml_type type) { return it != k_mod_map.end() ? &it->second : nullptr; } bool is_forbidden_tensor(const std::string& name) { - static const std::string kTokenEmbd{"token_embd.weight"}; - if (name == kTokenEmbd) return true; - //if (auto pos = name.find("attn_kv_b.weight"); pos != std::string::npos) return true; - return false; + return (name == "token_embd.weight" || name == "per_layer_token_embd.weight"); } } diff --git a/include/llama.h b/include/llama.h index ce5458335..05eb3b8fd 100644 --- a/include/llama.h +++ b/include/llama.h @@ -523,6 +523,7 @@ extern "C" { enum llama_ftype ftype; // quantize to this llama_ftype enum ggml_type output_tensor_type; // output tensor type enum ggml_type token_embedding_type; // token embeddings tensor type + enum ggml_type per_layer_token_embedding_type; // token embeddings tensor type enum ggml_type attn_q_type; // attention query tensor type enum ggml_type attn_k_type; // attention key tensor type enum ggml_type attn_v_type; // attention value tensor type diff --git a/src/llama-quantize.cpp b/src/llama-quantize.cpp index 574ab0ce8..0c12e40e6 100644 --- a/src/llama-quantize.cpp +++ b/src/llama-quantize.cpp @@ -409,6 +409,29 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n new_type = GGML_TYPE_IQ4_NL; } } + } else if (name == "per_layer_token_embd.weight") { + if (qs.params->per_layer_token_embedding_type < GGML_TYPE_COUNT) { + new_type = qs.params->per_layer_token_embedding_type; + } else { + if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || + ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || + ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS_R4 || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS_R4 || + ftype == LLAMA_FTYPE_MOSTLY_IQ1_S_R4 || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M_R4) { + new_type = GGML_TYPE_Q2_K; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M_R4) { + new_type = GGML_TYPE_IQ3_S; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_KT) { + new_type = GGML_TYPE_IQ3_S; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS_R4) { + new_type = GGML_TYPE_IQ3_K; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_BN || ftype == LLAMA_FTYPE_MOSTLY_IQ2_BN || ftype == LLAMA_FTYPE_MOSTLY_IQ2_BN_R4) { + new_type = GGML_TYPE_IQ4_NL; + } + } } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S_R4 || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M_R4) { if (name.find("attn_v.weight") != std::string::npos) { if (qs.model.hparams.n_expert >= 4 || qs.model.hparams.n_gqa() >= 4) new_type = GGML_TYPE_IQ4_K_R4; @@ -830,6 +853,15 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n } } + if (name == "per_layer_token_embd.weight") { + auto working_type = interleaved_properties(new_type).first; + if (working_type != new_type) { + printf("\n============ Per-layer Token embeddings cannot be quantized with row-interleaved quants\n"); + printf("---> Changed %s to %s\n", ggml_type_name(new_type), ggml_type_name(working_type)); + new_type = working_type; + } + } + return new_type; } @@ -1522,6 +1554,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (params->token_embedding_type < GGML_TYPE_COUNT && strcmp(tensor->name, "token_embd.weight") == 0) { new_type = params->token_embedding_type; } + if (params->per_layer_token_embedding_type < GGML_TYPE_COUNT && strcmp(tensor->name, "per_layer_token_embd.weight") == 0) { + new_type = params->per_layer_token_embedding_type; + } if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { new_type = params->output_tensor_type; } diff --git a/src/llama.cpp b/src/llama.cpp index 56c6b2bad..4987911c8 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -7186,34 +7186,35 @@ struct llama_context_params llama_context_default_params() { struct llama_model_quantize_params llama_model_quantize_default_params() { struct llama_model_quantize_params result = { - /*.nthread =*/ 0, - /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1, - /*.output_tensor_type =*/ GGML_TYPE_COUNT, - /*.token_embedding_type =*/ GGML_TYPE_COUNT, - /*.attn_q_type =*/ GGML_TYPE_COUNT, - /*.attn_k_type =*/ GGML_TYPE_COUNT, - /*.attn_v_type =*/ GGML_TYPE_COUNT, - /*.attn_qkv_type =*/ GGML_TYPE_COUNT, - /*.attn_output_type =*/ GGML_TYPE_COUNT, - /*.ffn_gate_type =*/ GGML_TYPE_COUNT, - /*.ffn_down_type =*/ GGML_TYPE_COUNT, - /*.ffn_up_type =*/ GGML_TYPE_COUNT, - /*.ffn_gat_inp_type =*/ GGML_TYPE_COUNT, - /*.extra_output_type =*/ GGML_TYPE_COUNT, - /*.allow_requantize =*/ false, - /*.quantize_output_tensor =*/ true, - /*.only_copy =*/ false, - /*.pure =*/ false, - /*.keep_split =*/ false, - /*.ignore_imatrix_rules =*/ false, - /*.only_repack =*/ false, - /*.dry_run =*/ false, - /*.partial_requant =*/ false, - /*.imatrix =*/ nullptr, - /*.kv_overrides =*/ nullptr, - /*.custom_quants =*/ nullptr, - /*.repack_pattern =*/ nullptr, - /*.user_data =*/ nullptr, + /*.nthread =*/ 0, + /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1, + /*.output_tensor_type =*/ GGML_TYPE_COUNT, + /*.token_embedding_type =*/ GGML_TYPE_COUNT, + /*.per_layer_token_embedding_type =*/ GGML_TYPE_COUNT, + /*.attn_q_type =*/ GGML_TYPE_COUNT, + /*.attn_k_type =*/ GGML_TYPE_COUNT, + /*.attn_v_type =*/ GGML_TYPE_COUNT, + /*.attn_qkv_type =*/ GGML_TYPE_COUNT, + /*.attn_output_type =*/ GGML_TYPE_COUNT, + /*.ffn_gate_type =*/ GGML_TYPE_COUNT, + /*.ffn_down_type =*/ GGML_TYPE_COUNT, + /*.ffn_up_type =*/ GGML_TYPE_COUNT, + /*.ffn_gat_inp_type =*/ GGML_TYPE_COUNT, + /*.extra_output_type =*/ GGML_TYPE_COUNT, + /*.allow_requantize =*/ false, + /*.quantize_output_tensor =*/ true, + /*.only_copy =*/ false, + /*.pure =*/ false, + /*.keep_split =*/ false, + /*.ignore_imatrix_rules =*/ false, + /*.only_repack =*/ false, + /*.dry_run =*/ false, + /*.partial_requant =*/ false, + /*.imatrix =*/ nullptr, + /*.kv_overrides =*/ nullptr, + /*.custom_quants =*/ nullptr, + /*.repack_pattern =*/ nullptr, + /*.user_data =*/ nullptr, }; return result;