furyhawk/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-21 10:15:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenize : align usage by using common args (#25516)

Browse Source 
Migrate the tokenize tool to common_params_parse, replacing its
hand-rolled argv parsing, Windows UTF-8 handling and file reading
with the shared common helpers.

Expose the model-sourcing flags (-m, -mu, -dr, -hf, -hff, --offline,
HF_TOKEN) to LLAMA_EXAMPLE_TOKENIZE, and register --ids, --stdin,
--no-bos, --no-parse-special and --show-count as common args.
parse_special defaults to true for TOKENIZE to preserve the old
behavior. Errors now go through LOG_ERR instead of fprintf(stderr).

Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët
2026-07-14 15:20:53 +02:00
committed by GitHub
parent a7312ae94f
commit 8ff8c4299d
3 changed files with 120 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+44 -7
View File
@@ -1179,6 +1179,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.sampling.temp = 0.2; // lower temp by default for better quality
} else if (ex == LLAMA_EXAMPLE_SERVER) {
params.n_parallel = -1; // auto by default
} else if (ex == LLAMA_EXAMPLE_TOKENIZE) {
params.parse_special = true; // parse special tokens by default, like the old tokenize tool
}
params.use_color = tty_can_use_colors();
@@ -2746,14 +2748,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.path = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
add_opt(common_arg(
{"-mu", "--model-url"}, "MODEL_URL",
"model download url (default: unused)",
[](common_params & params, const std::string & value) {
params.model.url = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL_URL"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL_URL"));
add_opt(common_arg(
{ "-dr", "--docker-repo" }, "[<repo>/]<model>[:quant]",
"Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.\n"
@@ -2762,7 +2764,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.docker_repo = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_DOCKER_REPO"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_DOCKER_REPO"));
add_opt(common_arg(
{"-hf", "-hfr", "--hf-repo"}, "<user>/<model>[:quant]",
"Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n"
@@ -2772,14 +2774,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.model.hf_repo = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_REPO"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_REPO"));
add_opt(common_arg(
{"-hff", "--hf-file"}, "FILE",
"Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)",
[](common_params & params, const std::string & value) {
params.model.hf_file = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_FILE"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE"));
add_opt(common_arg(
{"-hfv", "-hfrv", "--hf-repo-v"}, "<user>/<model>[:quant]",
"Hugging Face model repository for the vocoder model (default: unused)",
@@ -2800,7 +2802,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.hf_token = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("HF_TOKEN"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("HF_TOKEN"));
add_opt(common_arg(
{"--mtp"},
"also download the multi-token prediction (MTP) head, if available (default: unused)",
@@ -2916,6 +2918,41 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.parse_special = true;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--ids"},
string_format("only print the token IDs, in a Python-parseable list form like [1, 2, 3] (default: %s)", params.tokenize_ids ? "true" : "false"),
[](common_params & params) {
params.tokenize_ids = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--stdin"},
string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"),
[](common_params & params) {
params.tokenize_stdin = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--no-bos"},
string_format("do not add a BOS token to the prompt, even if the model normally uses one (default: %s)", params.tokenize_no_bos ? "true" : "false"),
[](common_params & params) {
params.tokenize_no_bos = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--no-parse-special"},
string_format("do not parse special tokens (chat, tool, etc) (default: %s)", !params.parse_special ? "true" : "false"),
[](common_params & params) {
params.parse_special = false;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"--show-count"},
string_format("print the total number of tokens (default: %s)", params.tokenize_show_count ? "true" : "false"),
[](common_params & params) {
params.tokenize_show_count = true;
}
).set_examples({LLAMA_EXAMPLE_TOKENIZE}));
add_opt(common_arg(
{"-pps"},
string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"),
@@ -3506,7 +3543,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params) {
params.offline = true;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE"));
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_OFFLINE"));
add_opt(common_arg(
{"-lv", "--verbosity", "--log-verbosity"}, "N",
string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n"
common/common.h
+7
View File
@@ -105,6 +105,7 @@ enum llama_example {
LLAMA_EXAMPLE_RESULTS,
LLAMA_EXAMPLE_EXPORT_GRAPH_OPS,
LLAMA_EXAMPLE_DOWNLOAD,
LLAMA_EXAMPLE_TOKENIZE,
LLAMA_EXAMPLE_COUNT,
};
@@ -716,6 +717,12 @@ struct common_params {
// batched-bench params
bool batched_bench_output_jsonl = false;
// tokenize params
bool tokenize_ids = false; // if true, only print the token IDs
bool tokenize_stdin = false; // if true, read the prompt from stdin
bool tokenize_no_bos = false; // if true, do not add the BOS token
bool tokenize_show_count = false; // if true, print the total token count
// common params
std::string out_file; // output filename for all example programs
// optional callback for model loading progress and cancellation:
tools/tokenize/tokenize.cpp
+69 -258
View File
@@ -1,5 +1,6 @@
#include "arg.h"
#include "common.h"
//#include "log.h" // TODO: start using log.h
#include "log.h"
#include "llama.h"
#include <clocale>
@@ -8,115 +9,22 @@
#include <fstream>
#include <string>
#include <vector>
#include <iostream> // TODO: remove me
#include <iostream>
#include <sstream>
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <shellapi.h> // For CommandLineToArgvW
#endif
static void print_usage_information(const char * argv0) {
printf("usage: %s [options]\n\n", argv0);
printf("The tokenize program tokenizes a prompt using a given model,\n");
printf("and prints the resulting tokens to standard output.\n\n");
printf("It needs a model file, a prompt, and optionally other flags\n");
printf("to control the behavior of the tokenizer.\n\n");
printf(" The possible options are:\n");
printf("\n");
printf(" -h, --help print this help and exit\n");
printf(" -m MODEL_PATH, --model MODEL_PATH path to model.\n");
printf(" --ids if given, only print numerical token IDs, and not token strings.\n");
printf(" The output format looks like [1, 2, 3], i.e. parseable by Python.\n");
printf(" -f PROMPT_FNAME, --file PROMPT_FNAME read prompt from a file.\n");
printf(" -p PROMPT, --prompt PROMPT read prompt from the argument.\n");
printf(" --stdin read prompt from standard input.\n");
printf(" --no-bos do not ever add a BOS token to the prompt, even if normally the model uses a BOS token.\n");
printf(" --no-escape do not escape input (such as \\n, \\t, etc.).\n");
printf(" --no-parse-special do not parse control tokens.\n");
printf(" --log-disable disable logs. Makes stderr quiet when loading the model.\n");
printf(" --show-count print the total number of tokens.\n");
}
static void print_usage(int argc, char ** argv) {
(void) argc;
static void llama_log_callback_null(ggml_log_level level, const char * text, void * user_data) {
(void) level;
(void) text;
(void) user_data;
}
static std::string read_prompt_from_file(const char * filepath, bool & success) {
success = false;
std::ifstream in(filepath, std::ios::binary);
if (!in) {
fprintf(stderr, "%s: could not open file '%s' for reading: %s\n", __func__, filepath, strerror(errno));
return std::string();
}
// do not assume the file is seekable (e.g. /dev/stdin)
std::stringstream buffer;
buffer << in.rdbuf();
if (in.fail()) {
fprintf(stderr, "%s: could not read the entire file '%s': %s\n", __func__, filepath, strerror(errno));
return std::string();
}
success = true;
return buffer.str();
}
//
// Function: ingest_args(...) -> vector<string>
//
// Takes argc and argv arguments, and converts them to a vector of UTF-8 encoded
// strings, as an STL vector<string>.
//
// In particular, it handles character encoding shenanigans on Windows.
//
// Note: raw_argc and raw_argv are not actually read at all on Windows.
// On Windows we call GetCommandLineW to get the arguments in wchar_t
// format, ignoring the regular argc/argv arguments to main().
//
// TODO: potential opportunity to roll common stuff into common/console.cpp
// in relation to Windows wchar_t shenanigans.
static std::vector<std::string> ingest_args(int raw_argc, char ** raw_argv) {
std::vector<std::string> argv;
// Handle Windows, if given non-ASCII arguments.
// We convert wchar_t arguments into UTF-8 char* on this platform.
// Lets you invoke 'tokenize' on Windows cmd.exe with non-ASCII characters
// without throwing tantrums.
#if defined(_WIN32)
int argc;
const LPWSTR cmdline_wargv = GetCommandLineW();
LPWSTR * wargv = CommandLineToArgvW(cmdline_wargv, &argc);
// silence unused arg warnings
(void) raw_argc;
(void) raw_argv;
for (int i = 0; i < argc; ++i) {
int length_needed = WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), 0, 0, NULL, NULL);
char * output_buf = (char *) calloc(length_needed+1, sizeof(char));
GGML_ASSERT(output_buf);
WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), output_buf, length_needed, NULL, NULL);
output_buf[length_needed] = '\0';
argv.push_back(output_buf);
free(output_buf);
}
LocalFree((HLOCAL) wargv);
#else
int argc = raw_argc;
for (int i = 0; i < argc; ++i) {
argv.push_back(raw_argv[i]);
}
#endif
GGML_ASSERT((unsigned int) argc == argv.size());
return argv;
LOG("\nexample usage:\n");
LOG("\n %s -m your_model.gguf -p \"Hello world\"\n", argv[0]);
LOG("\n %s -m your_model.gguf -f prompt.txt --ids\n", argv[0]);
LOG("\n cat prompt.txt | %s -m your_model.gguf --stdin --show-count\n", argv[0]);
LOG("\n");
}
//
@@ -184,166 +92,69 @@ static void write_utf8_cstr_to_stdout(const char * str, bool & invalid_utf8) {
#endif
}
int main(int raw_argc, char ** raw_argv) {
int main(int argc, char ** argv) {
std::setlocale(LC_NUMERIC, "C");
const std::vector<std::string> argv = ingest_args(raw_argc, raw_argv);
const int argc = argv.size();
common_params params;
if (argc <= 1) {
print_usage_information(argv[0].c_str());
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_TOKENIZE, print_usage)) {
return 1;
}
//////
// Read out all the command line arguments.
//////
// which prompt source was requested?
// -p/--prompt and -f/--file both end up in params.prompt (common's -f also
// strips a single trailing newline), but -f additionally records the path
// in params.prompt_file, so we use that to tell them apart.
const bool use_stdin = params.tokenize_stdin;
const bool use_file = !params.prompt_file.empty();
// variables where to put any arguments we see.
bool printing_ids = false;
bool no_bos = false;
bool no_escape = false;
bool no_parse_special = false;
bool disable_logging = false;
bool show_token_count = false;
const char * model_path = NULL;
const char * prompt_path = NULL;
const char * prompt_arg = NULL;
// track which arguments were explicitly given
// used for sanity checking down the line
bool model_path_set = false;
bool prompt_path_set = false;
bool prompt_set = false;
bool stdin_set = false;
int iarg = 1;
for (; iarg < argc; ++iarg) {
std::string arg{argv[iarg]};
if (arg == "-h" || arg == "--help") {
print_usage_information(argv[0].c_str());
return 0;
}
else if (arg == "--ids") {
printing_ids = true;
}
else if (arg == "-m" || arg == "--model") {
if (model_path_set) {
fprintf(stderr, "Error: -m or --model specified multiple times.\n");
return 1;
}
model_path = argv[++iarg].c_str();
model_path_set = true;
}
else if (arg == "--no-bos") {
no_bos = true;
}
else if (arg == "--no-escape") {
no_escape = true;
}
else if (arg == "--no-parse-special") {
no_parse_special = true;
}
else if (arg == "-p" || arg == "--prompt") {
if (prompt_set) {
fprintf(stderr, "Error: -p or --prompt specified multiple times.\n");
return 1;
}
prompt_arg = argv[++iarg].c_str();
prompt_set = true;
}
else if (arg == "-f" || arg == "--file") {
if (prompt_path_set) {
fprintf(stderr, "Error: -f or --file specified multiple times.\n");
return 1;
}
prompt_path = argv[++iarg].c_str();
prompt_path_set = true;
}
else if (arg == "--stdin") {
stdin_set = true;
}
else if (arg == "--log-disable") {
disable_logging = true;
}
else if (arg == "--show-count") {
show_token_count = true;
}
else {
fprintf(stderr, "Error: unknown option '%s'\n", argv[iarg].c_str());
return 1;
}
}
//////
// Sanity check the command line arguments.
//////
// Check that we have the required stuff set.
if (model_path_set && model_path == NULL) {
fprintf(stderr, "Error: --model requires an argument.\n");
return 1;
}
if (!model_path_set) {
fprintf(stderr, "Error: must specify --model.\n");
return 1;
}
if (prompt_path_set && prompt_path == NULL) {
fprintf(stderr, "Error: --file requires an argument.\n");
return 1;
}
if (prompt_set && prompt_arg == NULL) {
fprintf(stderr, "Error: --prompt requires an argument.\n");
return 1;
}
const int prompts_set = !!(prompt_path_set) + !!(prompt_set) + !!(stdin_set);
if (prompts_set > 1) {
fprintf(stderr, "Error: --stdin, --file and --prompt are mutually exclusive.\n");
return 1;
}
// Must have some prompt.
if (prompts_set == 0) {
fprintf(stderr, "Error: must specify one of: --stdin, --file or --prompt.\n");
// sanity check: --stdin is mutually exclusive with -f/--file and -p/--prompt
if (use_stdin && (use_file || !params.prompt.empty())) {
LOG_ERR("error: --stdin is mutually exclusive with --file and --prompt\n");
return 1;
}
GGML_ASSERT(model_path);
GGML_ASSERT(prompt_path || prompt_arg || stdin_set);
//////
// Figure out where will the prompt come from.
//////
// must have some prompt
if (!use_stdin && !use_file && params.prompt.empty()) {
LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n");
return 1;
}
std::string prompt;
if (prompt_path_set) {
bool success = false;
prompt = read_prompt_from_file(prompt_path, success);
if (!success) {
if (use_file) {
// read the file verbatim: common's -f handler strips a single trailing
// newline, but for a tokenizer the input bytes must be preserved exactly
// (a trailing newline is itself a token). escapes are applied locally
// to match the behavior of -p/--prompt and --stdin.
std::ifstream in(params.prompt_file, std::ios::binary);
if (!in) {
LOG_ERR("error: could not open file '%s' for reading\n", params.prompt_file.c_str());
return 1;
}
} else if (prompt_set) {
prompt = prompt_arg;
} else {
GGML_ASSERT(stdin_set);
// we read stdin *after* loading model (early exit if model cannot
// be loaded, which can be a nicer user experience)
}
//////
// Start actually doing the tokenizing stuff.
//////
if (disable_logging) {
llama_log_set(llama_log_callback_null, NULL);
std::stringstream ss;
ss << in.rdbuf();
prompt = ss.str();
if (params.escape) {
string_process_escapes(prompt);
}
} else if (!use_stdin) {
// -p/--prompt is already escape-processed by common_params_parse()
// (controlled by --escape/--no-escape), so use it verbatim here.
prompt = params.prompt;
}
// else: we read stdin *after* loading the model (early exit if the
// model cannot be loaded, which is a nicer user experience)
llama_backend_init();
// load only the vocabulary (no weights), since tokenizing does not need them
llama_model_params model_params = llama_model_default_params();
model_params.vocab_only = true;
llama_model * model = llama_model_load_from_file(model_path, model_params);
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);
if (!model) {
fprintf(stderr, "Error: could not load model from file '%s'.\n", model_path);
LOG_ERR("error: could not load model from file '%s'.\n", params.model.path.c_str());
return 1;
}
@@ -352,42 +163,41 @@ int main(int raw_argc, char ** raw_argv) {
llama_context_params ctx_params = llama_context_default_params();
llama_context * ctx = llama_init_from_model(model, ctx_params);
if (!ctx) {
fprintf(stderr, "Error: could not create context.\n");
LOG_ERR("error: could not create context.\n");
return 1;
}
// read entire prompt from stdin?
if (stdin_set) {
GGML_ASSERT(!prompt_path_set && !prompt_set);
if (params.tokenize_stdin) {
std::stringstream stdin_buffer;
stdin_buffer << std::cin.rdbuf();
if (std::cin.fail()) {
fprintf(stderr, "Error: could not read the entire standard input.\n");
LOG_ERR("error: could not read the entire standard input.\n");
return 1;
}
prompt = stdin_buffer.str();
// stdin is not seen by common_params_parse(), so apply escape handling
// here to match the behavior of -p/--prompt and -f/--file.
if (params.escape) {
string_process_escapes(prompt);
}
}
const bool model_wants_add_bos = llama_vocab_get_add_bos(vocab);
const bool add_bos = model_wants_add_bos && !no_bos;
const bool parse_special = !no_parse_special;
const bool escape = !no_escape;
if (escape) {
string_process_escapes(prompt);
}
const bool add_bos = model_wants_add_bos && !params.tokenize_no_bos;
const bool parse_special = params.parse_special;
std::vector<llama_token> tokens;
tokens = common_tokenize(vocab, prompt, add_bos, parse_special);
if (printing_ids) {
if (params.tokenize_ids) {
printf("[");
}
for (int i = 0; i < (int) tokens.size(); i++) {
if (printing_ids) {
if (params.tokenize_ids) {
if (i > 0) {
printf(", ");
}
@@ -404,13 +214,14 @@ int main(int raw_argc, char ** raw_argv) {
}
}
if (printing_ids) {
if (params.tokenize_ids) {
printf("]\n");
}
if (show_token_count) {
if (params.tokenize_show_count) {
printf("Total number of tokens: %zu\n", tokens.size());
}
// silence valgrind
llama_free(ctx);
llama_model_free(model);