diff --git a/common/arg.cpp b/common/arg.cpp index b6fddae00c..7bc770ab4f 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2932,7 +2932,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_TOKENIZE})); add_opt(common_arg( {"--stdin"}, - string_format("read the prompt from stdin (mutually exclusive with -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"), + string_format("read the prompt from stdin (takes precedence over -f/--file and -p/--prompt) (default: %s)", params.tokenize_stdin ? "true" : "false"), [](common_params & params) { params.tokenize_stdin = true; } diff --git a/tools/tokenize/tokenize.cpp b/tools/tokenize/tokenize.cpp index 23120ad2e1..77b33c4a46 100644 --- a/tools/tokenize/tokenize.cpp +++ b/tools/tokenize/tokenize.cpp @@ -103,19 +103,11 @@ int main(int argc, char ** argv) { return 1; } - // which prompt source was requested? - // -p/--prompt and -f/--file both end up in params.prompt (common's -f also - // strips a single trailing newline), but -f additionally records the path - // in params.prompt_file, so we use that to tell them apart. + // -f and -p both land in params.prompt; -f also sets prompt_file. -f and -p + // resolve like the other tools (no mutual exclusion), --stdin takes precedence. const bool use_stdin = params.tokenize_stdin; const bool use_file = !params.prompt_file.empty(); - // sanity check: --stdin is mutually exclusive with -f/--file and -p/--prompt - if (use_stdin && (use_file || !params.prompt.empty())) { - LOG_ERR("error: --stdin is mutually exclusive with --file and --prompt\n"); - return 1; - } - // must have some prompt if (!use_stdin && !use_file && params.prompt.empty()) { LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n");