diff --git a/common/arg.cpp b/common/arg.cpp index 9aefcd2029..9676adafe6 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1179,6 +1179,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.temp = 0.2; // lower temp by default for better quality } else if (ex == LLAMA_EXAMPLE_SERVER) { params.n_parallel = -1; // auto by default + } else if (ex == LLAMA_EXAMPLE_TOKENIZE) { + params.parse_special = true; // parse special tokens by default, like the old tokenize tool } params.use_color = tty_can_use_colors(); @@ -2746,14 +2748,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.path = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL")); add_opt(common_arg( {"-mu", "--model-url"}, "MODEL_URL", "model download url (default: unused)", [](common_params & params, const std::string & value) { params.model.url = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL_URL")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL_URL")); add_opt(common_arg( { "-dr", "--docker-repo" }, "[/][:quant]", "Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.\n" @@ -2762,7 +2764,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.docker_repo = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_DOCKER_REPO")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_DOCKER_REPO")); add_opt(common_arg( {"-hf", "-hfr", "--hf-repo"}, "/[:quant]", "Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n" @@ -2772,14 +2774,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.hf_repo = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_REPO")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_REPO")); add_opt(common_arg( {"-hff", "--hf-file"}, "FILE", "Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)", [](common_params & params, const std::string & value) { params.model.hf_file = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_FILE")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE")); add_opt(common_arg( {"-hfv", "-hfrv", "--hf-repo-v"}, "/[:quant]", "Hugging Face model repository for the vocoder model (default: unused)", @@ -2800,7 +2802,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.hf_token = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("HF_TOKEN")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("HF_TOKEN")); add_opt(common_arg( {"--mtp"}, "also download the multi-token prediction (MTP) head, if available (default: unused)", @@ -2916,6 +2918,41 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.parse_special = true; } ).set_examples({LLAMA_EXAMPLE_IMATRIX})); + add_opt(common_arg( + {"--ids"}, + string_format("only print the token IDs, in a Python-parseable list form like [1, 2, 3] (default: %s)", params.tokenize_ids ? "true" : "false"), + [](common_params & params) { + params.tokenize_ids = true; + } + ).set_examples({LLAMA_EXAMPLE_TOKENIZE})); + add_opt(common_arg( + {"--stdin"}, + string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"), + [](common_params & params) { + params.tokenize_stdin = true; + } + ).set_examples({LLAMA_EXAMPLE_TOKENIZE})); + add_opt(common_arg( + {"--no-bos"}, + string_format("do not add a BOS token to the prompt, even if the model normally uses one (default: %s)", params.tokenize_no_bos ? "true" : "false"), + [](common_params & params) { + params.tokenize_no_bos = true; + } + ).set_examples({LLAMA_EXAMPLE_TOKENIZE})); + add_opt(common_arg( + {"--no-parse-special"}, + string_format("do not parse special tokens (chat, tool, etc) (default: %s)", !params.parse_special ? "true" : "false"), + [](common_params & params) { + params.parse_special = false; + } + ).set_examples({LLAMA_EXAMPLE_TOKENIZE})); + add_opt(common_arg( + {"--show-count"}, + string_format("print the total number of tokens (default: %s)", params.tokenize_show_count ? "true" : "false"), + [](common_params & params) { + params.tokenize_show_count = true; + } + ).set_examples({LLAMA_EXAMPLE_TOKENIZE})); add_opt(common_arg( {"-pps"}, string_format("is the prompt shared across parallel sequences (default: %s)", params.is_pp_shared ? "true" : "false"), @@ -3506,7 +3543,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.offline = true; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_OFFLINE")); add_opt(common_arg( {"-lv", "--verbosity", "--log-verbosity"}, "N", string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n" diff --git a/common/common.h b/common/common.h index 7ed1a98275..66760005cd 100644 --- a/common/common.h +++ b/common/common.h @@ -105,6 +105,7 @@ enum llama_example { LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS, LLAMA_EXAMPLE_DOWNLOAD, + LLAMA_EXAMPLE_TOKENIZE, LLAMA_EXAMPLE_COUNT, }; @@ -716,6 +717,12 @@ struct common_params { // batched-bench params bool batched_bench_output_jsonl = false; + // tokenize params + bool tokenize_ids = false; // if true, only print the token IDs + bool tokenize_stdin = false; // if true, read the prompt from stdin + bool tokenize_no_bos = false; // if true, do not add the BOS token + bool tokenize_show_count = false; // if true, print the total token count + // common params std::string out_file; // output filename for all example programs // optional callback for model loading progress and cancellation: diff --git a/tools/tokenize/tokenize.cpp b/tools/tokenize/tokenize.cpp index 32cf8c8eb9..23120ad2e1 100644 --- a/tools/tokenize/tokenize.cpp +++ b/tools/tokenize/tokenize.cpp @@ -1,5 +1,6 @@ +#include "arg.h" #include "common.h" -//#include "log.h" // TODO: start using log.h +#include "log.h" #include "llama.h" #include @@ -8,115 +9,22 @@ #include #include #include -#include // TODO: remove me +#include +#include #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include -#include // For CommandLineToArgvW #endif -static void print_usage_information(const char * argv0) { - printf("usage: %s [options]\n\n", argv0); - printf("The tokenize program tokenizes a prompt using a given model,\n"); - printf("and prints the resulting tokens to standard output.\n\n"); - printf("It needs a model file, a prompt, and optionally other flags\n"); - printf("to control the behavior of the tokenizer.\n\n"); - printf(" The possible options are:\n"); - printf("\n"); - printf(" -h, --help print this help and exit\n"); - printf(" -m MODEL_PATH, --model MODEL_PATH path to model.\n"); - printf(" --ids if given, only print numerical token IDs, and not token strings.\n"); - printf(" The output format looks like [1, 2, 3], i.e. parseable by Python.\n"); - printf(" -f PROMPT_FNAME, --file PROMPT_FNAME read prompt from a file.\n"); - printf(" -p PROMPT, --prompt PROMPT read prompt from the argument.\n"); - printf(" --stdin read prompt from standard input.\n"); - printf(" --no-bos do not ever add a BOS token to the prompt, even if normally the model uses a BOS token.\n"); - printf(" --no-escape do not escape input (such as \\n, \\t, etc.).\n"); - printf(" --no-parse-special do not parse control tokens.\n"); - printf(" --log-disable disable logs. Makes stderr quiet when loading the model.\n"); - printf(" --show-count print the total number of tokens.\n"); -} +static void print_usage(int argc, char ** argv) { + (void) argc; -static void llama_log_callback_null(ggml_log_level level, const char * text, void * user_data) { - (void) level; - (void) text; - (void) user_data; -} - -static std::string read_prompt_from_file(const char * filepath, bool & success) { - success = false; - - std::ifstream in(filepath, std::ios::binary); - if (!in) { - fprintf(stderr, "%s: could not open file '%s' for reading: %s\n", __func__, filepath, strerror(errno)); - return std::string(); - } - // do not assume the file is seekable (e.g. /dev/stdin) - std::stringstream buffer; - buffer << in.rdbuf(); - if (in.fail()) { - fprintf(stderr, "%s: could not read the entire file '%s': %s\n", __func__, filepath, strerror(errno)); - return std::string(); - } - - success = true; - return buffer.str(); -} - -// -// Function: ingest_args(...) -> vector -// -// Takes argc and argv arguments, and converts them to a vector of UTF-8 encoded -// strings, as an STL vector. -// -// In particular, it handles character encoding shenanigans on Windows. -// -// Note: raw_argc and raw_argv are not actually read at all on Windows. -// On Windows we call GetCommandLineW to get the arguments in wchar_t -// format, ignoring the regular argc/argv arguments to main(). -// -// TODO: potential opportunity to roll common stuff into common/console.cpp -// in relation to Windows wchar_t shenanigans. -static std::vector ingest_args(int raw_argc, char ** raw_argv) { - std::vector argv; - - // Handle Windows, if given non-ASCII arguments. - // We convert wchar_t arguments into UTF-8 char* on this platform. - // Lets you invoke 'tokenize' on Windows cmd.exe with non-ASCII characters - // without throwing tantrums. -#if defined(_WIN32) - int argc; - const LPWSTR cmdline_wargv = GetCommandLineW(); - LPWSTR * wargv = CommandLineToArgvW(cmdline_wargv, &argc); - - // silence unused arg warnings - (void) raw_argc; - (void) raw_argv; - - for (int i = 0; i < argc; ++i) { - int length_needed = WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), 0, 0, NULL, NULL); - char * output_buf = (char *) calloc(length_needed+1, sizeof(char)); - GGML_ASSERT(output_buf); - - WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), output_buf, length_needed, NULL, NULL); - output_buf[length_needed] = '\0'; - - argv.push_back(output_buf); - free(output_buf); - } - - LocalFree((HLOCAL) wargv); -#else - int argc = raw_argc; - for (int i = 0; i < argc; ++i) { - argv.push_back(raw_argv[i]); - } -#endif - - GGML_ASSERT((unsigned int) argc == argv.size()); - - return argv; + LOG("\nexample usage:\n"); + LOG("\n %s -m your_model.gguf -p \"Hello world\"\n", argv[0]); + LOG("\n %s -m your_model.gguf -f prompt.txt --ids\n", argv[0]); + LOG("\n cat prompt.txt | %s -m your_model.gguf --stdin --show-count\n", argv[0]); + LOG("\n"); } // @@ -184,166 +92,69 @@ static void write_utf8_cstr_to_stdout(const char * str, bool & invalid_utf8) { #endif } -int main(int raw_argc, char ** raw_argv) { +int main(int argc, char ** argv) { std::setlocale(LC_NUMERIC, "C"); - const std::vector argv = ingest_args(raw_argc, raw_argv); - const int argc = argv.size(); + common_params params; - if (argc <= 1) { - print_usage_information(argv[0].c_str()); + common_init(); + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_TOKENIZE, print_usage)) { return 1; } - ////// - // Read out all the command line arguments. - ////// + // which prompt source was requested? + // -p/--prompt and -f/--file both end up in params.prompt (common's -f also + // strips a single trailing newline), but -f additionally records the path + // in params.prompt_file, so we use that to tell them apart. + const bool use_stdin = params.tokenize_stdin; + const bool use_file = !params.prompt_file.empty(); - // variables where to put any arguments we see. - bool printing_ids = false; - bool no_bos = false; - bool no_escape = false; - bool no_parse_special = false; - bool disable_logging = false; - bool show_token_count = false; - const char * model_path = NULL; - const char * prompt_path = NULL; - const char * prompt_arg = NULL; - - // track which arguments were explicitly given - // used for sanity checking down the line - bool model_path_set = false; - bool prompt_path_set = false; - bool prompt_set = false; - bool stdin_set = false; - - int iarg = 1; - for (; iarg < argc; ++iarg) { - std::string arg{argv[iarg]}; - if (arg == "-h" || arg == "--help") { - print_usage_information(argv[0].c_str()); - return 0; - } - else if (arg == "--ids") { - printing_ids = true; - } - else if (arg == "-m" || arg == "--model") { - if (model_path_set) { - fprintf(stderr, "Error: -m or --model specified multiple times.\n"); - return 1; - } - model_path = argv[++iarg].c_str(); - model_path_set = true; - } - else if (arg == "--no-bos") { - no_bos = true; - } - else if (arg == "--no-escape") { - no_escape = true; - } - else if (arg == "--no-parse-special") { - no_parse_special = true; - } - else if (arg == "-p" || arg == "--prompt") { - if (prompt_set) { - fprintf(stderr, "Error: -p or --prompt specified multiple times.\n"); - return 1; - } - prompt_arg = argv[++iarg].c_str(); - prompt_set = true; - } - else if (arg == "-f" || arg == "--file") { - if (prompt_path_set) { - fprintf(stderr, "Error: -f or --file specified multiple times.\n"); - return 1; - } - prompt_path = argv[++iarg].c_str(); - prompt_path_set = true; - } - else if (arg == "--stdin") { - stdin_set = true; - } - else if (arg == "--log-disable") { - disable_logging = true; - } - else if (arg == "--show-count") { - show_token_count = true; - } - else { - fprintf(stderr, "Error: unknown option '%s'\n", argv[iarg].c_str()); - return 1; - } - } - - ////// - // Sanity check the command line arguments. - ////// - - // Check that we have the required stuff set. - if (model_path_set && model_path == NULL) { - fprintf(stderr, "Error: --model requires an argument.\n"); - return 1; - } - if (!model_path_set) { - fprintf(stderr, "Error: must specify --model.\n"); - return 1; - } - if (prompt_path_set && prompt_path == NULL) { - fprintf(stderr, "Error: --file requires an argument.\n"); - return 1; - } - if (prompt_set && prompt_arg == NULL) { - fprintf(stderr, "Error: --prompt requires an argument.\n"); - return 1; - } - const int prompts_set = !!(prompt_path_set) + !!(prompt_set) + !!(stdin_set); - if (prompts_set > 1) { - fprintf(stderr, "Error: --stdin, --file and --prompt are mutually exclusive.\n"); - return 1; - } - // Must have some prompt. - if (prompts_set == 0) { - fprintf(stderr, "Error: must specify one of: --stdin, --file or --prompt.\n"); + // sanity check: --stdin is mutually exclusive with -f/--file and -p/--prompt + if (use_stdin && (use_file || !params.prompt.empty())) { + LOG_ERR("error: --stdin is mutually exclusive with --file and --prompt\n"); return 1; } - GGML_ASSERT(model_path); - GGML_ASSERT(prompt_path || prompt_arg || stdin_set); - - ////// - // Figure out where will the prompt come from. - ////// + // must have some prompt + if (!use_stdin && !use_file && params.prompt.empty()) { + LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n"); + return 1; + } std::string prompt; - if (prompt_path_set) { - bool success = false; - prompt = read_prompt_from_file(prompt_path, success); - if (!success) { + if (use_file) { + // read the file verbatim: common's -f handler strips a single trailing + // newline, but for a tokenizer the input bytes must be preserved exactly + // (a trailing newline is itself a token). escapes are applied locally + // to match the behavior of -p/--prompt and --stdin. + std::ifstream in(params.prompt_file, std::ios::binary); + if (!in) { + LOG_ERR("error: could not open file '%s' for reading\n", params.prompt_file.c_str()); return 1; } - } else if (prompt_set) { - prompt = prompt_arg; - } else { - GGML_ASSERT(stdin_set); - // we read stdin *after* loading model (early exit if model cannot - // be loaded, which can be a nicer user experience) - } - - ////// - // Start actually doing the tokenizing stuff. - ////// - - if (disable_logging) { - llama_log_set(llama_log_callback_null, NULL); + std::stringstream ss; + ss << in.rdbuf(); + prompt = ss.str(); + if (params.escape) { + string_process_escapes(prompt); + } + } else if (!use_stdin) { + // -p/--prompt is already escape-processed by common_params_parse() + // (controlled by --escape/--no-escape), so use it verbatim here. + prompt = params.prompt; } + // else: we read stdin *after* loading the model (early exit if the + // model cannot be loaded, which is a nicer user experience) llama_backend_init(); + // load only the vocabulary (no weights), since tokenizing does not need them llama_model_params model_params = llama_model_default_params(); model_params.vocab_only = true; - llama_model * model = llama_model_load_from_file(model_path, model_params); + llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params); if (!model) { - fprintf(stderr, "Error: could not load model from file '%s'.\n", model_path); + LOG_ERR("error: could not load model from file '%s'.\n", params.model.path.c_str()); return 1; } @@ -352,42 +163,41 @@ int main(int raw_argc, char ** raw_argv) { llama_context_params ctx_params = llama_context_default_params(); llama_context * ctx = llama_init_from_model(model, ctx_params); if (!ctx) { - fprintf(stderr, "Error: could not create context.\n"); + LOG_ERR("error: could not create context.\n"); return 1; } // read entire prompt from stdin? - if (stdin_set) { - GGML_ASSERT(!prompt_path_set && !prompt_set); - + if (params.tokenize_stdin) { std::stringstream stdin_buffer; stdin_buffer << std::cin.rdbuf(); if (std::cin.fail()) { - fprintf(stderr, "Error: could not read the entire standard input.\n"); + LOG_ERR("error: could not read the entire standard input.\n"); return 1; } prompt = stdin_buffer.str(); + + // stdin is not seen by common_params_parse(), so apply escape handling + // here to match the behavior of -p/--prompt and -f/--file. + if (params.escape) { + string_process_escapes(prompt); + } } const bool model_wants_add_bos = llama_vocab_get_add_bos(vocab); - const bool add_bos = model_wants_add_bos && !no_bos; - const bool parse_special = !no_parse_special; - const bool escape = !no_escape; - - if (escape) { - string_process_escapes(prompt); - } + const bool add_bos = model_wants_add_bos && !params.tokenize_no_bos; + const bool parse_special = params.parse_special; std::vector tokens; tokens = common_tokenize(vocab, prompt, add_bos, parse_special); - if (printing_ids) { + if (params.tokenize_ids) { printf("["); } for (int i = 0; i < (int) tokens.size(); i++) { - if (printing_ids) { + if (params.tokenize_ids) { if (i > 0) { printf(", "); } @@ -404,13 +214,14 @@ int main(int raw_argc, char ** raw_argv) { } } - if (printing_ids) { + if (params.tokenize_ids) { printf("]\n"); } - if (show_token_count) { + if (params.tokenize_show_count) { printf("Total number of tokens: %zu\n", tokens.size()); } + // silence valgrind llama_free(ctx); llama_model_free(model);