furyhawk/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-21 10:15:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

mtmd: deepseek-ocr v1 multi-tile (#24717)

Browse Source 
* mtmd: deepseek-ocr v1 multi-tile dynamic resolution + unified image-preprocessors for both versions (ds-ocr v1 and v2)

* remove hacky API

* fuse row into a long image

* almost working

* adapt to new preprocessor api

* rm debugging printf

* improve

* mtmd: dsocr-tiles fixes (#25481)

* ds-ocr img-preproc fuse_row tile-drop fix for multi rows and columns images

* mtmd drop the duplicate redundant img_end

* deepseekocr graph simplify CLS broadcast cleanup

* test-deepseek-ocr: relax v1 single-view tolerance; drop trailing prompt space; make DRY opt-in and n_predict model-specific (#25486)

---------

Co-authored-by: Saba Fallah <10401143+sfallah@users.noreply.github.com>
Co-authored-by: Saba Fallah <sabafallah@gmail.com>
This commit is contained in:
Xuan-Son Nguyen
2026-07-10 16:05:49 +02:00
committed by GitHub
co-authored by Saba Fallah Saba Fallah
parent 07d9378286
commit 3e706dd55f
10 changed files with 239 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-graph.h
+8 -2
View File
@@ -20,8 +20,8 @@ struct clip_graph {
const clip_hparams & hparams;
projector_type proj_type;
// we only support single image per batch
const clip_image_f32 & img;
const clip_image_f32 & img; // for backward compat
const clip_image_f32_batch * img_batch = nullptr;
const int patch_size;
const int n_patches_x;
@@ -63,6 +63,12 @@ struct clip_graph {
//
void cb(ggml_tensor * cur0, const char * name, int il) const;
const clip_image_f32 & get_img(size_t idx) const {
GGML_ASSERT(img_batch);
GGML_ASSERT(idx < img_batch->entries.size());
return img_batch->entries[idx];
}
// siglip2 naflex
ggml_tensor * resize_position_embeddings(uint32_t interpolation_mode = DEFAULT_INTERPOLATION_MODE);
tools/mtmd/clip-model.h
+1
View File
@@ -69,6 +69,7 @@ struct clip_hparams {
std::vector<clip_image_size> image_res_candidates;
int32_t preproc_min_tiles = 0;
int32_t preproc_max_tiles = 0;
int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr)
resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
resize_algo image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)
tools/mtmd/clip.cpp
+29 -5
View File
@@ -1024,6 +1024,8 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
GGML_ABORT("missing cgraph builder");
}
builder->img_batch = &imgs;
// TODO [QWEN_VIDEO]: improve this in the future
builder->n_batch = imgs.entries.size();
@@ -1580,7 +1582,16 @@ struct clip_model_loader {
get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
get_u32(KEY_SAM_N_EMBD, hparams.sam_n_embd, true);
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
hparams.preproc_min_tiles = 2;
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR) {
hparams.preproc_max_tiles = 9;
hparams.preproc_tile_size = 640;
// the CLIP/ViT body runs its layernorms at 1e-5 (the SAM stage uses 1e-6)
hparams.eps = 1e-5f;
}
if (model.proj_type == PROJECTOR_TYPE_DEEPSEEKOCR2) {
hparams.preproc_max_tiles = 6;
hparams.preproc_tile_size = 768;
// qwen2 encoder is GQA, requires KEY_N_HEAD_KV
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
}
@@ -3251,6 +3262,9 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
return (img->nx() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
return img->nx() / (params.patch_size * params.n_merge);
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
return (img->nx() / params.patch_size) / 4;
default:
break;
}
@@ -3460,10 +3474,17 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
// E.g., 64x64 -> 16x16 patches
n_patches /= 16;
// build_global_local_features adds image newlines and view separator
// Formula: h*(w+1) + 1 where h = w = sqrt(n_patches)
int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
n_patches = h * (h + 1) + 1;
if (img->add_viewsep) {
// global view: one image-newline per token-row + trailing view separator
const int h = static_cast<int>(std::sqrt(static_cast<float>(n_patches)));
n_patches = h * (h + 1) + 1;
} else if (img->ny() >= img->nx() && img->ny() % img->nx() == 0) {
// tile row: one image-newline per token-row
const int grid_w = img->ny() / img->nx();
const int tile_patches = img->nx() / (patch_size * 4); // patches per tile side (SAM divides by 4)
const int h = tile_patches;
n_patches = (tile_patches * grid_w + 1) * h;
}
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
@@ -4103,7 +4124,10 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
GGML_ASSERT(pos_w == pos_h);
GGML_ASSERT(
(pos_w == pos_h) // overview image
|| (pos_h >= pos_w && pos_h % pos_w == 0) // tile images
);
const int window = hparams.attn_window_size;
const int pos = pos_w;
tools/mtmd/models/deepseekocr.cpp
+87 -17
View File
@@ -96,6 +96,8 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
const int n_heads = hparams.sam_n_head;
const int d_heads = n_embd / n_heads;
const int window = hparams.attn_window_size;
// SAM stage runs its layernorms at 1e-6
const float sam_eps = 1e-6f;
ggml_tensor * inpL;
@@ -134,7 +136,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
ggml_tensor * shortcut = cur;
// layernorm1
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, sam_eps, il);
const int64_t w0 = cur->ne[1];
const int64_t h0 = cur->ne[2];
@@ -214,7 +216,7 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
ggml_tensor * inpFF = cur;
// layernorm2
cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, sam_eps, il);
// ffn
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b,
@@ -229,12 +231,12 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
cur = ggml_conv_2d(ctx0, model.neck_0_w, cur, 1, 1, 0, 0, 1, 1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, hparams.eps, -1);
cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, sam_eps, -1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
cur = ggml_conv_2d(ctx0, model.neck_2_w, cur, 1, 1, 1, 1, 1, 1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, hparams.eps, -1);
cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, sam_eps, -1);
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
cur = ggml_conv_2d(ctx0, model.net_2, cur, 2, 2, 1, 1, 1, 1);
@@ -248,8 +250,40 @@ ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
ggml_cgraph * clip_graph_deepseekocr::build() {
// patch embedding
ggml_tensor * inp_raw = build_inp_raw();
bool is_overview = img.add_viewsep;
int n_tiles_per_row = 0;
// note: we expect either a batch of rows or a batch of overviews, but not a mix of both
if (!is_overview) {
// handle the case where we have a batch of rows
// sanity check
for (auto & entry : img_batch->entries) {
if (entry.add_viewsep) {
throw std::runtime_error("DeepSeek-OCR: mixed overview and non-overview images in batch");
}
if (entry.nx() != img.nx() || entry.ny() != img.ny()) {
throw std::runtime_error("DeepSeek-OCR: mixed image sizes in batch");
}
}
GGML_ASSERT(img.ny() >= img.nx());
GGML_ASSERT(img.ny() % img.nx() == 0);
n_tiles_per_row = img.ny() / img.nx();
// input shape: [tile_size, tile_size * n_tiles_per_row, 3]
// we want to reshape it to [tile_size, tile_size, 3, n_tiles_per_row]
inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), n_tiles_per_row, 3);
inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 1, 3, 2));
}
ggml_tensor * sam_out = build_sam(inp_raw);
if (!is_overview) {
n_batch = n_tiles_per_row;
}
const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];
ggml_tensor * clip_out;
@@ -257,7 +291,9 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
{
ggml_tensor * inp;
inp = ggml_reshape_2d(ctx0, sam_out, clip_n_patches, sam_out->ne[2]);
// sam_out: [patch_h, patch_w, n_embd, n_batch]
// -> [n_embd, clip_n_patches, n_batch]
inp = ggml_reshape_3d(ctx0, sam_out, clip_n_patches, sam_out->ne[2], sam_out->ne[3]);
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
ggml_tensor * new_pos_embd = model.position_embeddings;
@@ -281,8 +317,11 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
n_pos = tgt_size * tgt_size + 1;
}
// add CLS token
inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
// add CLS token per batch item
// inp: [n_embd, clip_n_patches, n_batch]
// class_embedding: [n_embd] -> [n_embd, 1, n_batch]
ggml_tensor * cls_embd = ggml_repeat_4d(ctx0, model.class_embedding, n_embd, 1, n_batch, 1);
inp = ggml_concat(ctx0, cls_embd, inp, 1);
// for selecting learned pos embd, used by ViT
ggml_tensor * positions = ggml_cast(ctx0, ggml_arange(ctx0, 0, n_pos, 1), GGML_TYPE_I32);
@@ -294,25 +333,56 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
clip_out = cur;
}
// sam_out: [patch_h, patch_w, n_embd, n_batch]
// -> [n_embd, clip_n_patches, n_batch]
sam_out = ggml_cont(ctx0, ggml_permute(ctx0, sam_out, 1, 2, 0, 3));
sam_out = ggml_reshape_2d(ctx0, sam_out, sam_out->ne[0], clip_n_patches);
clip_out = ggml_view_2d(ctx0, clip_out, n_embd, clip_n_patches, clip_out->nb[1], clip_out->nb[1]);
sam_out = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0], clip_n_patches, n_batch);
// clip_out: [n_embd, n_pos, n_batch] where n_pos = clip_n_patches + 1 (CLS)
// strip CLS token: skip first position, view only the patch tokens
clip_out = ggml_view_3d(ctx0, clip_out, n_embd, clip_n_patches, n_batch,
clip_out->nb[1], clip_out->nb[2], clip_out->nb[1]);
ggml_tensor * cur;
cur = ggml_concat(ctx0, clip_out, sam_out, 0);
cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
cur = ggml_add(ctx0, cur, model.mm_fc_b);
const auto h = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
const auto w = h;
const auto n_dim = cur->ne[0];
if (is_overview) {
// global view: weave one newline per row + trailing view separator
const auto h = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
const auto w = h;
const auto n_dim = cur->ne[0];
ggml_tensor * imgnl;
ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, h*(w+1) + 1)
} else {
// tile row: interleave tiles within each row, add newline per row
const int grid_x = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
const int grid_y = grid_x;
const auto n_dim = cur->ne[0];
imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, 1);
cur = ggml_reshape_3d(ctx0, cur, n_dim, w, h);
cur = ggml_reshape_2d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h);
cur = ggml_concat(ctx0, cur, model.view_seperator, 1); // (n_dim, h*(w+1) + 1)
// (n_dim, clip_n_patches, n_batch) -> (n_dim, grid_x, grid_y, n_batch)
cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x, grid_y, n_batch);
// tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
// to A.row0 B.row0 A.row1 B.row1 ...
// then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
// interleave tiles: (n_dim, grid_x, grid_y, n_batch) -> (n_dim, grid_x, n_batch, grid_y)
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
// merge: (n_dim, grid_x, n_batch, grid_y) -> (n_dim, grid_x*n_batch, grid_y, 1)
cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_batch, grid_y, 1);
// append newline per row: (n_dim, grid_x*n_batch+1, grid_y, 1)
ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, 1);
cur = ggml_concat(ctx0, cur, imgnl, 1);
// flatten: (n_dim, (grid_x*n_batch+1)*grid_y)
cur = ggml_reshape_2d(ctx0, cur, n_dim, (grid_x * n_batch + 1) * grid_y);
}
cb(cur, "dsocr_output", -1);
tools/mtmd/models/models.h
+1
View File
@@ -127,6 +127,7 @@ struct clip_graph_deepseekocr : clip_graph {
clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
// bool support_batch() const override { return true; } // TODO: support batch for DeepSeek-OCR v1
};
struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
tools/mtmd/mtmd-image.cpp
+54 -61
View File
@@ -1107,44 +1107,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_i
// mtmd_image_preprocessor_deepseekocr
//
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ };
// TODO: support 512 (tiny) and 640 (small) once we have eval data for them
const int64_t orig_area = static_cast<int64_t>(img.get_size().area());
size_t mode_i = 0;
int64_t min_diff = std::numeric_limits<int64_t>::max();
for (size_t i = 0; i < std::size(native_resolutions); i++) {
const int64_t r = native_resolutions[i];
const int64_t diff = std::abs(orig_area - r * r);
if (diff < min_diff) {
mode_i = i;
min_diff = diff;
}
}
const int image_size = native_resolutions[mode_i];
// Aspect-preserving fit-and-pad. Pillow bicubic + PAD_NEAREST for
// byte-parity with the upstream deepseek-ai/DeepSeek-OCR HF preprocessor.
clip_image_u8 padded;
img_tool::resize(img, padded, {image_size, image_size}, RESIZE_ALGO_BICUBIC_PILLOW,
PAD_NEAREST, hparams.image_pad_color);
mtmd_image_preproc_out output;
output.append_overview(hparams, padded, true);
output.grid_x = 0;
output.grid_y = 0;
// TODO @ngxson : support slicing for DeepSeek-OCR, to do in another PR
return output;
}
//
// mtmd_image_preprocessor_deepseekocr2
//
// candidate tile grids (cols, rows) with min_tiles <= cols*rows <= max_tiles
// sorted by tile count
std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr2::get_target_ratios() {
std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr::get_target_ratios() const {
std::vector<clip_image_size> ratios;
for (int n = min_tiles; n <= max_tiles; n++) {
for (int w = 1; w <= n; w++) {
@@ -1171,13 +1134,11 @@ std::vector<clip_image_size> mtmd_image_preprocessor_deepseekocr2::get_target_ra
return ratios;
}
// pick the grid whose aspect ratio is closest to the image
// on a tie, prefer the larger grid when the image fits
clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio(
clip_image_size mtmd_image_preprocessor_deepseekocr::find_closest_aspect_ratio(
float aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width,
int height) {
int height) const {
float best_ratio_diff = std::numeric_limits<float>::max();
clip_image_size best_ratio = { 1, 1 };
const float area = static_cast<float>(width * height);
@@ -1198,37 +1159,69 @@ clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio(
return best_ratio;
}
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img) {
// emit 768x768 local tiles when the image is larger than a tile in either
// dimension, then always a 1024x1024 global view. order: [tiles..., global].
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) {
mtmd_image_preproc_out output;
int grid_w = 0;
int grid_h = 0;
const auto img_size = img.get_size();
// global view: aspect-preserving fit-and-pad to base_size
clip_image_u8 padded;
img_tool::resize(img, padded,
{ base_size, base_size },
RESIZE_ALGO_BICUBIC_PILLOW,
PAD_NEAREST,
hparams.image_pad_color);
output.append_overview(hparams, padded, true);
output.overview.add_viewsep = true;
// if this condition doesn't hold, the output is overview only, no tiles
if (img_size.width > tile_size || img_size.height > tile_size) {
const float aspect_ratio = static_cast<float>(img_size.width) / img_size.height;
const auto target_ratios = get_target_ratios();
const clip_image_size grid = find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
const clip_image_size grid =
find_closest_aspect_ratio(aspect_ratio, target_ratios, img_size.width, img_size.height);
grid_w = grid.width;
grid_h = grid.height;
// stretch onto the grid (no aspect preserve), then crop tiles row-major.
clip_image_u8 refined;
img_tool::resize(img, refined, { tile_size * grid.width, tile_size * grid.height },
RESIZE_ALGO_BICUBIC_PILLOW, PAD_NONE);
img_tool::resize(img, refined, { tile_size * grid_w, tile_size * grid_h }, RESIZE_ALGO_BICUBIC_PILLOW,
PAD_NONE);
for (int row = 0; row < grid.height; row++) {
for (int col = 0; col < grid.width; col++) {
clip_image_u8 tile;
img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
output.append(hparams, tile, true);
for (int row = 0; row < grid_h; row++) {
if (fuse_row) {
// concat all tiles in this row into a single image, along the H axis
// output image size: w = tile_size, h = tile_size * grid_w
// this is to ensure the whole row is always processed together
clip_image_u8 row_img;
row_img.set_size({tile_size, tile_size * grid_w}, false);
for (int col = 0; col < grid_w; col++) {
for (int py = 0; py < tile_size; py++) {
for (int px = 0; px < tile_size; px++) {
row_img.set_pixel(px, col * tile_size + py,
refined.get_pixel(col * tile_size + px, row * tile_size + py));
}
}
}
output.append(hparams, row_img, true);
} else {
for (int col = 0; col < grid_w; col++) {
clip_image_u8 tile;
img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size);
output.append(hparams, tile, true);
}
}
}
if (fuse_row) {
grid_w = 1; // each fused row is one image; a single output column
}
}
// global view: aspect-preserving fit-and-pad to base_size.
clip_image_u8 padded;
img_tool::resize(img, padded, { base_size, base_size }, RESIZE_ALGO_BICUBIC_PILLOW,
PAD_NEAREST, hparams.image_pad_color);
output.append_overview(hparams, padded, true);
output.overview.add_viewsep = true;
LOG_DBG("%s: grid size: %d x %d (%d tiles) + global view\n", __func__, grid_w, grid_h, grid_w * grid_h);
LOG_DBG("%s: overview size: %d x %d\n", __func__, padded.get_size().width, padded.get_size().height);
output.grid_x = grid_w;
output.grid_y = grid_h;
return output;
}
tools/mtmd/mtmd-image.h
+19 -19
View File
@@ -160,29 +160,29 @@ struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
// DeepSeek-OCR (v1/v2) global view + optional local tile grid
struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
// DeepSeek-OCR-2: a 1024x1024 global view, plus InternVL-style 768x768 local
// tiles when the image is larger than a tile in either dimension.
struct mtmd_image_preprocessor_deepseekocr2 : mtmd_image_preprocessor {
static constexpr int base_size = 1024; // global view
static constexpr int tile_size = 768; // local tile
static constexpr int min_tiles = 2;
static constexpr int max_tiles = 6;
mtmd_image_preprocessor_deepseekocr2(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx)
: mtmd_image_preprocessor(ctx),
fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR),
base_size(hparams.image_size),
tile_size(hparams.preproc_tile_size),
min_tiles(hparams.preproc_min_tiles),
max_tiles(hparams.preproc_max_tiles) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
private:
static std::vector<clip_image_size> get_target_ratios();
static clip_image_size find_closest_aspect_ratio(
float aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width,
int height);
bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
int base_size; // global view
int tile_size; // each tile
int min_tiles;
int max_tiles;
std::vector<clip_image_size> get_target_ratios() const;
clip_image_size find_closest_aspect_ratio(
float aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width, int height) const;
};
// custom image preprocessing for Step3VL
tools/mtmd/mtmd.cpp
+2 -7
View File
@@ -618,15 +618,10 @@ struct mtmd_context {
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR:
{
img_end = "\n"; // prevent empty batch on llama-server
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
ov_img_first = false;
} break;
case PROJECTOR_TYPE_DEEPSEEKOCR2:
{
img_end = "\n"; // prevent empty batch on llama-server
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr2>(ctx_v);
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);
ov_img_first = false;
} break;
case PROJECTOR_TYPE_HUNYUANVL:
@@ -1132,6 +1127,7 @@ struct mtmd_tokenizer {
// add slices (or tiles)
if (!chunks.empty()) {
LOG_DBG("%s: adding %d slices (%d rows x %d cols)\n", __func__, (int)chunks.size(), n_row, n_col);
GGML_ASSERT((int)chunks.size() == n_row * n_col);
add_text(ctx->tok_slices_start);
for (int y = 0; y < n_row; y++) {
@@ -1174,7 +1170,6 @@ struct mtmd_tokenizer {
cur.entries.emplace_back(std::move(ov_chunk));
add_text(ctx->tok_ov_img_end);
}
} else {
if (preproc_out.entries.size() == 0) {
tools/mtmd/tests/test-1-positive.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 225 KiB

tools/mtmd/tests/test-deepseek-ocr.py
+38 -8
View File
@@ -29,12 +29,15 @@ class ModelSpec:
mmproj_arg: str
model_default: str
mmproj_default: str
prompt: str = "Free OCR. "
prompt: str = "Free OCR."
n_predict: int = 512
n_ctx: int | None = None
# Unlimited-OCR's "document parsing" prompt emits <|det|> grounding markup that
# the HF reference strips in result.md; drop it before scoring to match.
strip_grounding: bool = False
# v2/Unlimited loop on hard tiles; DRY caps it the way HF's
# no_repeat_ngram_size does. v1 scores fine without it.
dry: bool = False
@dataclass
@@ -69,6 +72,9 @@ MODELS = {
model_arg="--llama-model-2", mmproj_arg="--mmproj-2",
model_default="gguf_models/deepseek-ai/deepseek-ocr-2-bf16.gguf",
mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-2-bf16.gguf",
# v2 keeps generating past 512 on multi-tile; give it room to match the HF ref.
n_predict=2048,
dry=True,
),
"unlimited": ModelSpec(
key="unlimited", label="Unlimited-OCR",
@@ -83,6 +89,7 @@ MODELS = {
n_predict=4096,
n_ctx=16384,
strip_grounding=True,
dry=True,
),
}
@@ -91,7 +98,9 @@ CASES = [
model_key="v1", label="single-view scan",
image="tools/mtmd/test-1.jpeg",
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
hf_cer=0.3030, hf_chrf=67.52, cer_tol=0.02, chrf_tol=2.0,
# Fragile image: the HF ref itself swings ~0.286-0.314 across precision
# configs -- hence the wide tol. llama.cpp bf16 ~0.322/63.8.
hf_cer=0.3140, hf_chrf=67.57, cer_tol=0.04, chrf_tol=5.0,
),
TestCase(
model_key="v2", label="single-view scan",
@@ -103,6 +112,24 @@ CASES = [
# is one pixel off and lands at ~0.69 instead.
hf_cer=0.7761, hf_chrf=28.70, cer_tol=0.12, chrf_tol=8.0,
),
TestCase(
model_key="v1", label="multi-tile (dynamic resolution)",
image="tools/mtmd/tests/test-1-positive.png",
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
# 429x806 -- 806 > 640 triggers the v1 "Gundam" path: (1,2) grid ->
# 2 local 640 tiles + 1 global 1024 view. Regression guard for the
# tiling preprocessor -- a broken tile path craters the score.
# hf_cer/hf_chrf are HF v1's measured scores -- it reads this clean crop exactly.
hf_cer=0.0000, hf_chrf=100.00, cer_tol=0.03, chrf_tol=3.0,
),
TestCase(
model_key="v2", label="multi-tile (dynamic resolution)",
image="tools/mtmd/tests/test-1-positive.png",
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
# 429x806 -- 806 > 768 triggers the v2 path: (1,2) grid ->
# 2 local 768 tiles + 1 global 1024 view = 545 image tokens.
hf_cer=0.0236, hf_chrf=97.05, cer_tol=0.03, chrf_tol=3.0,
),
TestCase(
model_key="unlimited", label="single-view scan",
image="tools/mtmd/test-1.jpeg",
@@ -180,14 +207,17 @@ def run_mtmd_cli(spec: "ModelSpec", model_path, mmproj_path, image_path, bin_pat
"--flash-attn", "off", # match the HF "eager" attention reference
"--no-warmup",
"-n", str(spec.n_predict), # cap loops on hard images (KV would otherwise fill)
]
if spec.dry:
# HF decodes with no_repeat_ngram_size; llama.cpp's analog is DRY.
# Default DRY breakers include "\n", so they are cleared below.
"--dry-multiplier", "0.8",
"--dry-base", "1.75",
"--dry-allowed-length", "2",
"--dry-penalty-last-n", "-1",
"--dry-sequence-breaker", "none",
]
cmd += [
"--dry-multiplier", "0.8",
"--dry-base", "1.75",
"--dry-allowed-length", "2",
"--dry-penalty-last-n", "-1",
"--dry-sequence-breaker", "none",
]
if spec.n_ctx is not None:
cmd += ["-c", str(spec.n_ctx)]
logger.debug(f" command: {' '.join(cmd)}")