furyhawk/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-21 10:15:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenize : align usage by using common args (#25516)

Browse Source 
Migrate the tokenize tool to common_params_parse, replacing its
hand-rolled argv parsing, Windows UTF-8 handling and file reading
with the shared common helpers.

Expose the model-sourcing flags (-m, -mu, -dr, -hf, -hff, --offline,
HF_TOKEN) to LLAMA_EXAMPLE_TOKENIZE, and register --ids, --stdin,
--no-bos, --no-parse-special and --show-count as common args.
parse_special defaults to true for TOKENIZE to preserve the old
behavior. Errors now go through LOG_ERR instead of fprintf(stderr).

Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët
2026-07-14 15:20:53 +02:00
committed by GitHub
parent a7312ae94f
commit 8ff8c4299d
3 changed files with 120 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+44 -7
View File
@@ -1179,6 +1179,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.sampling.temp = 0.2; // lower temp by default for better quality
} else if (ex == LLAMA_EXAMPLE_SERVER) {
params.n_parallel = -1; // auto by default
} else if (ex == LLAMA_EXAMPLE_TOKENIZE) {
params.parse_special = true; // parse special tokens by default, like the old tokenize tool
}
params.use_color = tty_can_use_colors();
@@ -2746,14 +2748,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.path = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
add_opt(common_arg(
{"-mu", "--model-url"}, "MODEL_URL",
"model download url (default: unused)",
[](common_params & params, const std::string & value) {
params.model.url = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL_URL"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL_URL"));
add_opt(common_arg(
{ "-dr", "--docker-repo" }, "[<repo>/]<model>[:quant]",
"Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.\n"
@@ -2762,7 +2764,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.docker_repo = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_DOCKER_REPO"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_DOCKER_REPO"));
add_opt(common_arg(
{"-hf", "-hfr", "--hf-repo"}, "<user>/<model>[:quant]",
"Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n"
@@ -2772,14 +2774,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.hf_repo = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_REPO"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_REPO"));
add_opt(common_arg(
{"-hff", "--hf-file"}, "FILE",
"Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)",
[](common_params & params, const std::string & value) {
params.model.hf_file = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_FILE"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE"));
add_opt(common_arg(
{"-hfv", "-hfrv", "--hf-repo-v"}, "<user>/<model>[:quant]",
"Hugging Face model repository for the vocoder model (default: unused)",
@@ -2800,7 +2802,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.hf_token = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("HF_TOKEN"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("HF_TOKEN"));
add_opt(common_arg(
{"--mtp"},
"also download the multi-token prediction (MTP) head, if available (default: unused)",
@@ -2916,6 +2918,41 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.parse_special = true;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--ids"},
string_format("only print the token IDs, in a Python-parseable list form like [1, 2, 3] (default: %s)", params.tokenize_ids ? "true" : "false"),
[](common_params & params) {
params.tokenize_ids = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--stdin"},
string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),
[](common_params & params) {
params.tokenize_stdin = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--no-bos"},
string_format("do not add a BOS token to the prompt, even if the model normally uses one (default: %s)", params.tokenize_no_bos ? "true" : "false"),
[](common_params & params) {
params.tokenize_no_bos = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--no-parse-special"},
string_format("do not parse special tokens (chat, tool, etc) (default: %s)", !params.parse_special ? "true" : "false"),
[](common_params & params) {
params.parse_special = false;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--show-count"},
string_format("print the total number of tokens (default: %s)", params.tokenize_show_count ? "true" : "false"),
[](common_params & params) {
params.tokenize_show_count = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"-pps"},
string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"),
@@ -3506,7 +3543,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params) {
params.offline = true;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_OFFLINE"));
add_opt(common_arg(
{"-lv", "--verbosity", "--log-verbosity"}, "N",
string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n"
common/common.h
+7
View File
@@ -105,6 +105,7 @@ enum llama_example {
LLAMA_EXAMPLE_RESULTS,
LLAMA_EXAMPLE_EXPORT_GRAPH_OPS,
LLAMA_EXAMPLE_DOWNLOAD,
LLAMA_EXAMPLE_TOKENIZE,
LLAMA_EXAMPLE_COUNT,
};
@@ -716,6 +717,12 @@ struct common_params {
// batched-bench params
bool batched_bench_output_jsonl = false;
// tokenize params
bool tokenize_ids = false; // if true, only print the token IDs
bool tokenize_stdin = false; // if true, read the prompt from stdin
bool tokenize_no_bos = false; // if true, do not add the BOS token
bool tokenize_show_count = false; // if true, print the total token count
// common params
std::string out_file; // output filename for all example programs
// optional callback for model loading progress and cancellation: