mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-07-21 02:05:51 +00:00
convert : fix dflash target tokenizer mismatch during conversion (#25733)
* spec: fix dflash target tokenizer mismatch during conversion * fix ci ty check
This commit is contained in:
+15
-1
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -641,7 +643,19 @@ class DFlashModel(Qwen3Model):
|
||||
logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}")
|
||||
original_dir = self.dir_model
|
||||
self.dir_model = self.target_model_dir
|
||||
super().set_vocab()
|
||||
|
||||
# Reuse the target model's own vocab handler (e.g. Gemma-4 needs its
|
||||
# own tokenizer logic, not the Qwen default).
|
||||
from . import get_model_class
|
||||
with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:
|
||||
target_arch = json.load(f)["architectures"][0]
|
||||
target_cls = get_model_class(target_arch)
|
||||
|
||||
if target_cls is not type(self):
|
||||
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
|
||||
else:
|
||||
super().set_vocab()
|
||||
|
||||
self.dir_model = original_dir
|
||||
|
||||
mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id")
|
||||
|
||||
Reference in New Issue
Block a user