mirror of
https://github.com/furyhawk/home_stack.git
synced 2026-07-21 10:16:47 +00:00
141 KiB
141 KiB
To run this, press "Runtime" and press "Run all" on a free Tesla T4 Google Colab instance!
To install Unsloth on your local device, follow our guide. This notebook is licensed LGPL-3.0.
You will learn how to do data prep, how to train, how to run the model, & how to save it
Introducing Unsloth Studio - a new open source, no-code web UI to train and run LLMs. Blog • Notebook
 Train models — no code needed |
 Run GGUF models on Mac, Windows & Linux |
Train MoEs - DeepSeek, GLM, Qwen and gpt-oss 12x faster with 35% less VRAM. Blog
Ultra Long-Context Reinforcement Learning is here with 7x more context windows! Blog
New in Reinforcement Learning: FP8 RL • Vision RL • Standby • gpt-oss RL
Visit our docs for all our model uploads and notebooks.
In [ ]:
%%capture
import os, importlib.util
!pip install --upgrade -qqq uv
if importlib.util.find_spec("torch") is None or "COLAB_" in "".join(os.environ.keys()):
try: import numpy, PIL; _numpy = f"numpy=={numpy.__version__}"; _pil = f"pillow=={PIL.__version__}"
except: _numpy = "numpy"; _pil = "pillow"
!uv pip install -qqq \
"torch==2.7.1" "triton>=3.3.0" {_numpy} {_pil} torchvision bitsandbytes "transformers==4.56.2" \
"unsloth_zoo[base] @ git+https://github.com/unslothai/unsloth-zoo" \
"unsloth[base] @ git+https://github.com/unslothai/unsloth"
!uv pip install -qqq --no-deps "torchcodec==0.5"
elif importlib.util.find_spec("unsloth") is None:
!uv pip install -qqq unsloth
!uv pip install --upgrade --no-deps transformers==4.56.2 "tokenizers>=0.22.0,<=0.23.0" trl==0.22.2 unsloth unsloth_zoo
# Mamba is supported only on torch==2.7.1. If you have newer torch versions, please wait 30 minutes!
!uv pip install --no-build-isolation mamba_ssm==2.2.5 causal_conv1d==1.5.2
!uv pip install --no-deps --upgrade "torchao>=0.16.0"In [ ]:
from unsloth import FastLanguageModel
import torch
fourbit_models = [
"unsloth/granite-4.0-micro",
"unsloth/granite-4.0-h-micro",
"unsloth/granite-4.0-h-tiny",
"unsloth/granite-4.0-h-small",
# Base pretrained Granite 4 models
"unsloth/granite-4.0-micro-base",
"unsloth/granite-4.0-h-micro-base",
"unsloth/granite-4.0-h-tiny-base",
"unsloth/granite-4.0-h-small-base",
# 4bit dynamic quants for superior accuracy and low memory use
"unsloth/gemma-3-12b-it-unsloth-bnb-4bit",
"unsloth/Phi-4",
"unsloth/Llama-3.1-8B",
"unsloth/Llama-3.2-3B",
"unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit" # [NEW] We support TTS models!
] # More models at https://huggingface.co/unsloth
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/granite-4.0-h-micro",
max_seq_length = 2048, # Choose any for long context!
load_in_4bit = False, # 4 bit quantization to reduce memory
load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory
full_finetuning = False, # [NEW!] We have full finetuning now!
)==((====))== Unsloth 2025.9.11: Fast Granitemoehybrid patching. Transformers: 4.55.4. \\ /| Tesla T4. Num GPUs = 1. Max memory: 14.741 GB. Platform: Linux. O^O/ \_/ \ Torch: 2.8.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.4.0 \ / Bfloat16 = FALSE. FA [Xformers = 0.0.32.post2. FA2 = False] "-____-" Free license: http://github.com/unslothai/unsloth Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored! Unsloth: QLoRA and full finetuning all not selected. Switching to 16bit LoRA.
The fast path for GraniteMoeHybrid will be used when running the model on a GPU
Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]
In [ ]:
model = FastLanguageModel.get_peft_model(
model,
r = 32, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
"gate_proj", "up_proj", "down_proj",
"shared_mlp.input_linear", "shared_mlp.output_linear"],
lora_alpha = 32,
lora_dropout = 0, # Supports any, but = 0 is optimized
bias = "none", # Supports any, but = "none" is optimized
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
random_state = 3407,
use_rslora = False, # We support rank stabilized LoRA
loftq_config = None, # And LoftQ
)Unsloth: Making `model.base_model.model.model` require gradients
In [ ]:
from datasets import load_dataset, Dataset
# Use the below shared sheet
# sheet_url = 'https://docs.google.com/spreadsheets/d/1NrjI5AGNIwRtKTAse5TW_hWq2CwAS03qCHif6vaaRh0/export?format=csv&gid=0'
# Or unsloth/Support-Bot-Recommendation
sheet_url = "https://huggingface.co/datasets/unsloth/Support-Bot-Recommendation/raw/main/support_recs.csv"
dataset = load_dataset(
"csv",
data_files = {"train": sheet_url},
column_names = ["snippet", "recommendation"], # Replace with the actual column names of your sheet
skiprows = 1 # skip header rows
)["train"]Downloading data: 0%| | 0.00/1.04M [00:00<?, ?B/s]
Generating train split: 0 examples [00:00, ? examples/s]
In [ ]:
def formatting_prompts_func(examples):
user_texts = examples['snippet']
response_texts = examples['recommendation']
messages = [
[{"role": "user", "content": user_text},
{"role": "assistant", "content": response_text}] for user_text, response_text in zip(user_texts, response_texts)
]
texts = [tokenizer.apply_chat_template(message, tokenize = False, add_generation_prompt = False) for message in messages]
return { "text" : texts, }
dataset = dataset.map(formatting_prompts_func, batched = True,)Map: 0%| | 0/504 [00:00<?, ? examples/s]
In [ ]:
dataset[5]["snippet"]'User: I\'m getting an error when trying to log in. \nAgent: What error message are you seeing? \nUser: It says "Invalid credentials" even though I\'m sure my password is correct. \nAgent: Have you tried clearing your browser cache? \nUser: Yes, I cleared it already. \nAgent: Let me check your account status. \nUser: I\'ve been using this account for months without issues. \nAgent: I found no issues with your account. \nUser: Maybe there\'s a problem with the login server? \nAgent: Let\'s try resetting your password. \nUser: I just did that, and it\'s not working either. \nAgent: I\'ll need to escalate this to our engineering team. \nUser: Okay, what should I do in the meantime? \nAgent: Try using a different browser or device. \nUser: I\'ll try Chrome on my laptop. \nAgent: Let me know if that resolves the issue.'
In [ ]:
dataset[5]['recommendation']'#### Analysis\nThe user is experiencing persistent login issues ("Invalid credentials", password reset failure) despite clearing cache and confirming correct credentials. No account anomalies were detected by the agent. The root cause remains unresolved and potentially related to server-side authentication or user-specific credential handling.\n\n#### Recommendation\n- Step 1: Confirm if using Chrome on the laptop resolved the login issue. (User action)\n- Step 2: If Step 1 was successful, no further immediate action needed. If not, proceed to escalate based on user feedback.\n- *Next best action for the agent*: Report back to the user whether using Chrome on their laptop confirmed or failed to resolve the issue.'In [ ]:
dataset[5]["text"]'<|start_of_role|>user<|end_of_role|>User: I\'m getting an error when trying to log in. \nAgent: What error message are you seeing? \nUser: It says "Invalid credentials" even though I\'m sure my password is correct. \nAgent: Have you tried clearing your browser cache? \nUser: Yes, I cleared it already. \nAgent: Let me check your account status. \nUser: I\'ve been using this account for months without issues. \nAgent: I found no issues with your account. \nUser: Maybe there\'s a problem with the login server? \nAgent: Let\'s try resetting your password. \nUser: I just did that, and it\'s not working either. \nAgent: I\'ll need to escalate this to our engineering team. \nUser: Okay, what should I do in the meantime? \nAgent: Try using a different browser or device. \nUser: I\'ll try Chrome on my laptop. \nAgent: Let me know if that resolves the issue.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>#### Analysis\nThe user is experiencing persistent login issues ("Invalid credentials", password reset failure) despite clearing cache and confirming correct credentials. No account anomalies were detected by the agent. The root cause remains unresolved and potentially related to server-side authentication or user-specific credential handling.\n\n#### Recommendation\n- Step 1: Confirm if using Chrome on the laptop resolved the login issue. (User action)\n- Step 2: If Step 1 was successful, no further immediate action needed. If not, proceed to escalate based on user feedback.\n- *Next best action for the agent*: Report back to the user whether using Chrome on their laptop confirmed or failed to resolve the issue.<|end_of_text|>\n'In [ ]:
from trl import SFTTrainer, SFTConfig
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset,
eval_dataset = None, # Can set up evaluation!
args = SFTConfig(
dataset_text_field = "text",
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4, # Use GA to mimic batch size!
warmup_steps = 5,
# num_train_epochs = 1, # Set this for 1 full training run.
max_steps = 60,
learning_rate = 2e-4, # Reduce to 2e-5 for long training runs
logging_steps = 1,
optim = "adamw_8bit",
weight_decay = 0.001,
lr_scheduler_type = "linear",
seed = 3407,
report_to = "none", # Use TrackIO/WandB etc
),
)Unsloth: We found double BOS tokens - we shall remove one automatically.
Unsloth: Tokenizing ["text"] (num_proc=6): 0%| | 0/504 [00:00<?, ? examples/s]
In [ ]:
from unsloth.chat_templates import train_on_responses_only
trainer = train_on_responses_only(
trainer,
instruction_part = "<|start_of_role|>user<|end_of_role|>",
response_part = "<|start_of_role|>assistant<|end_of_role|>",
)Map (num_proc=2): 0%| | 0/504 [00:00<?, ? examples/s]
In [ ]:
tokenizer.decode(trainer.train_dataset[100]["input_ids"])'<|start_of_role|>user<|end_of_role|>User: My account is locked. I tried to log in but got an error message saying "Too many failed attempts".\n\nAgent: Can you please try logging in again and enter the security code sent to your email? That should unlock your account temporarily.\n\nUser: I did that already. I received the code and entered it, but my account is still locked.\n\nAgent: I see. Have you tried resetting your password via the \'Forgot Password\' link?\n\nUser: Yes, I clicked on that. It sent me an email with a reset link, but when I tried to reset my password, I got an error message saying "Invalid request".\n\nAgent: Okay, I can\'t access your account to check directly. Could you please provide me with your account ID or email address associated with the account?\n\nUser: My email is user@example.com. Account ID is 123456789.\n\nAgent: Thank you. I\'m looking into this. It seems there might be an issue with the account lockout mechanism or the password reset process. I\'ll need to contact our security team to manually unlock your account and investigate further. I\'ll keep you updated on the progress.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>#### Analysis\nThe user is experiencing account lockout due to multiple failed login attempts, and standard troubleshooting steps like password reset and security code entry are failing, indicating a potential issue with the account lockout mechanism or password recovery system.\n\n#### Recommendation\n- Step 1: Attempt to reset the password using the \'Forgot Password\' link and provide the error details received.\n- Step 2: Contact support with the account ID/email and request manual account unlock and investigation.\n- *Next best action for the agent*: Instruct the user to contact support immediately, providing their account details for manual intervention and further investigation.<|end_of_text|>\n'
In [ ]:
tokenizer.decode([tokenizer.pad_token_id if x == -100 else x for x in trainer.train_dataset[100]["labels"]]).replace(tokenizer.pad_token, " ")" #### Analysis\nThe user is experiencing account lockout due to multiple failed login attempts, and standard troubleshooting steps like password reset and security code entry are failing, indicating a potential issue with the account lockout mechanism or password recovery system.\n\n#### Recommendation\n- Step 1: Attempt to reset the password using the 'Forgot Password' link and provide the error details received.\n- Step 2: Contact support with the account ID/email and request manual account unlock and investigation.\n- *Next best action for the agent*: Instruct the user to contact support immediately, providing their account details for manual intervention and further investigation.<|end_of_text|>\n"
In [ ]:
# @title Show current memory stats
gpu_stats = torch.cuda.get_device_properties(0)
start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
print(f"{start_gpu_memory} GB of memory reserved.")GPU = Tesla T4. Max memory = 14.741 GB. 6.059 GB of memory reserved.
In [ ]:
trainer_stats = trainer.train()==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1 \\ /| Num examples = 504 | Num Epochs = 1 | Total steps = 60 O^O/ \_/ \ Batch size per device = 2 | Gradient accumulation steps = 4 \ / Data Parallel GPUs = 1 | Total batch size (2 x 4 x 1) = 8 "-____-" Trainable parameters = 1,703,936 of 3,193,100,032 (0.05% trained)
Unsloth: Will smartly offload gradients to save VRAM!
[60/60 06:38, Epoch 0/1]
| Step | Training Loss |
|---|---|
| 1 | 0.893200 |
| 2 | 1.193700 |
| 3 | 1.385300 |
| 4 | 1.949300 |
| 5 | 1.909700 |
| 6 | 2.066900 |
| 7 | 1.923100 |
| 8 | 2.053300 |
| 9 | 1.503300 |
| 10 | 1.325700 |
| 11 | 1.352200 |
| 12 | 1.807200 |
| 13 | 1.629000 |
| 14 | 1.233600 |
| 15 | 1.699000 |
| 16 | 1.547100 |
| 17 | 1.678400 |
| 18 | 1.548900 |
| 19 | 1.197600 |
| 20 | 1.353900 |
| 21 | 1.487300 |
| 22 | 1.335600 |
| 23 | 1.281100 |
| 24 | 1.560200 |
| 25 | 1.750500 |
| 26 | 1.331500 |
| 27 | 1.068000 |
| 28 | 1.248100 |
| 29 | 1.445800 |
| 30 | 1.722400 |
| 31 | 0.849400 |
| 32 | 1.297800 |
| 33 | 1.238500 |
| 34 | 1.273300 |
| 35 | 1.082300 |
| 36 | 1.168300 |
| 37 | 1.298700 |
| 38 | 1.045200 |
| 39 | 1.244500 |
| 40 | 1.201900 |
| 41 | 1.276200 |
| 42 | 1.212700 |
| 43 | 1.011200 |
| 44 | 1.159400 |
| 45 | 1.310300 |
| 46 | 1.207200 |
| 47 | 1.025800 |
| 48 | 1.230800 |
| 49 | 1.118100 |
| 50 | 1.113900 |
| 51 | 1.386500 |
| 52 | 1.237000 |
| 53 | 1.429900 |
| 54 | 1.125200 |
| 55 | 0.882900 |
| 56 | 1.148100 |
| 57 | 1.617100 |
| 58 | 0.499600 |
| 59 | 1.432000 |
| 60 | 1.132300 |
In [ ]:
# @title Show final memory and time stats
used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
used_memory_for_lora = round(used_memory - start_gpu_memory, 3)
used_percentage = round(used_memory / max_memory * 100, 3)
lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)
print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.")
print(
f"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training."
)
print(f"Peak reserved memory = {used_memory} GB.")
print(f"Peak reserved memory for training = {used_memory_for_lora} GB.")
print(f"Peak reserved memory % of max memory = {used_percentage} %.")
print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.")954.0727 seconds used for training. 15.9 minutes used for training. Peak reserved memory = 10.42 GB. Peak reserved memory for training = 4.361 GB. Peak reserved memory % of max memory = 70.687 %. Peak reserved memory for training % of max memory = 29.584 %.
In [ ]:
# @title Test Scenarios
# --- Scenario 1: Video-Conferencing Screen-Share Bug (11 turns) ---
scenario_1 = """
User: Everyone in my meeting just sees a black screen when I share.
Agent: Sorry about that—are you sharing a window or your entire screen?
User: Entire screen on macOS Sonoma.
Agent: Thanks. Do you have “Enable hardware acceleration” toggled on in Settings → Video?
User: Yeah, that switch is on.
Agent: Could you try toggling it off and start a quick test share?
User: Did that—still black for attendees.
Agent: Understood. Are you on the desktop app v5.4.2 or the browser client?
User: Desktop v5.4.2—just updated this morning.
"""
# --- Scenario 2: Smart-Lock Low-Battery Loop (9 turns) ---
scenario_2 = """
User: I changed the batteries, but the lock app still says 5 % and won’t auto-lock.
Agent: Let’s check firmware. In the app, go to Settings → Device Info—what version shows?
User: 3.18.0-alpha.
Agent: Latest stable is 3.17.5. Did you enroll in the beta program?
User: I might have months ago.
Agent: Beta builds sometimes misreport battery. Remove one battery, wait ten seconds, reinsert, and watch the LED pattern.
User: LED blinks blue twice, then red once.
Agent: That blink code means “config mismatch.” Do you still have the old batteries handy?
User: Tossed them already.
"""
# --- Scenario 3: Accounting SaaS — Corrupted Invoice Export (10 turns) ---
scenario_3 = """
User: Every invoice I download today opens as a blank PDF.
Agent: Is this happening to historic invoices, new ones, or both?
User: Both. Anything I export is 0 bytes.
Agent: Are you exporting through “Bulk Actions” or individual invoice pages?
User: Individual pages.
Agent: Which browser/OS combo?
User: Chrome on Windows 11, latest update.
Agent: We released a new PDF renderer at 10 a.m. UTC. Could you try Edge quickly, just to rule out a caching quirk?
User: Tried Edge—same zero-byte file.
"""
# --- Scenario 4: Fitness-Tracker App — Stuck Step Count (8 turns) ---
scenario_4 = """
User: My step count has been frozen at 4,237 since last night.
Agent: Which phone are you syncing with?
User: iPhone 15, iOS 17.5.
Agent: In the Health Permissions screen, does “Motion & Fitness” show as ON?
User: Yes, it’s toggled on.
Agent: When you pull down to refresh the dashboard, does the sync spinner appear?
User: Spinner flashes for a second, then nothing changes.
"""
# --- Scenario 5: Online-Course Platform — Quiz Submission Error (12 turns) ---
scenario_5 = """
User: My quiz submits but then shows “Unknown grading error” and resets the answers.
Agent: Which course and quiz name?
User: History 301, Unit 2 Quiz.
Agent: Do you notice a red banner or any code like GR-### in the corner?
User: Banner says “GR-412”.
Agent: That code points to answer-payload size. Were you pasting images or long text into any answers?
User: Maybe a long essay—about 800 words in Question 5.
Agent: Are you on a laptop or mobile?
User: Laptop, Safari on macOS.
"""In [ ]:
FastLanguageModel.for_inference(model) # Enable native 2x faster inference
messages = [
{"role": "user", "content": scenario_1},
]
inputs = tokenizer.apply_chat_template(
messages,
tokenize = True,
add_generation_prompt = True, # Must add for generation
padding = True,
return_tensors = "pt",
return_dict = True,
).to("cuda")
from transformers import TextStreamer
text_streamer = TextStreamer(tokenizer, skip_prompt = False)
_ = model.generate(**inputs,
streamer = text_streamer,
max_new_tokens = 512, # Increase if tokens are getting cut off
use_cache = True,
# Adjust the sampling params to your preference
do_sample = True,
temperature = 0.7, top_p = 0.8, top_k = 20,
)<|start_of_role|>user<|end_of_role|> User: Everyone in my meeting just sees a black screen when I share. Agent: Sorry about that—are you sharing a window or your entire screen? User: Entire screen on macOS Sonoma. Agent: Thanks. Do you have “Enable hardware acceleration” toggled on in Settings → Video? User: Yeah, that switch is on. Agent: Could you try toggling it off and start a quick test share? User: Did that—still black for attendees. Agent: Understood. Are you on the desktop app v5.4.2 or the browser client? User: Desktop v5.4.2—just updated this morning. <|end_of_text|> <|start_of_role|>assistant<|end_of_role|>#### Analysis - The user reports a black screen for attendees when sharing the entire screen on macOS Sonoma. - The user has "Enable hardware acceleration" enabled in Settings → Video. - The user is on the desktop app v5.4.2, which was updated this morning. #### Recommendation - Step 1: Ask the user to restart the desktop app and try sharing again. - Step 2: If the issue persists, ask the user to uninstall the desktop app, restart their computer, and reinstall the latest version of the desktop app. - *Next best action for the agent*: Ask the user to perform the restart and reinstall steps and confirm if the issue is resolved.<|end_of_text|>
In [ ]:
FastLanguageModel.for_inference(model) # Enable native 2x faster inference
messages = [
{"role": "user", "content": scenario_2},
]
inputs = tokenizer.apply_chat_template(
messages,
tokenize = True,
add_generation_prompt = True, # Must add for generation
padding = True,
return_tensors = "pt",
return_dict = True,
).to("cuda")
from transformers import TextStreamer
text_streamer = TextStreamer(tokenizer, skip_prompt = False)
_ = model.generate(**inputs,
streamer = text_streamer,
max_new_tokens = 512, # Increase if tokens are getting cut off
use_cache = True,
# Adjust the sampling params to your preference
do_sample = False,
temperature = 0.7, top_p = 0.8, top_k = 20,
)<|start_of_role|>user<|end_of_role|> User: I changed the batteries, but the lock app still says 5 % and won’t auto-lock. Agent: Let’s check firmware. In the app, go to Settings → Device Info—what version shows? User: 3.18.0-alpha. Agent: Latest stable is 3.17.5. Did you enroll in the beta program? User: I might have months ago. Agent: Beta builds sometimes misreport battery. Remove one battery, wait ten seconds, reinsert, and watch the LED pattern. User: LED blinks blue twice, then red once. Agent: That blink code means “config mismatch.” Do you still have the old batteries handy? User: Tossed them already. <|end_of_text|> <|start_of_role|>assistant<|end_of_role|>#### Analysis The user's lock app shows 5% battery and fails to auto-lock, despite recent battery replacement. The firmware version is 3.18.0-alpha, which is outdated. The LED blink pattern indicates a configuration mismatch, likely due to the user's previous beta enrollment. #### Recommendation 1. **Verify firmware version** in the app's Device Info section. 2. **Check for beta enrollment** history. 3. **Remove one battery**, wait ten seconds, then reinsert it. 4. **Observe the LED pattern** for confirmation of the configuration mismatch error. 5. **Provide the user with the correct firmware version** (3.17.5) and instructions to update. 6. **Ask the user to re-enroll** in the beta program if applicable, or confirm they have the latest stable firmware.<|end_of_text|>
In [ ]:
model.save_pretrained("granite_lora") # Local saving
tokenizer.save_pretrained("granite_lora")
# model.push_to_hub("your_name/granite_lora", token = "YOUR_HF_TOKEN") # Online saving
# tokenizer.push_to_hub("your_name/granite_lora", token = "YOUR_HF_TOKEN") # Online saving('lora_model/tokenizer_config.json',
'lora_model/special_tokens_map.json',
'lora_model/chat_template.jinja',
'lora_model/vocab.json',
'lora_model/merges.txt',
'lora_model/added_tokens.json',
'lora_model/tokenizer.json')In [ ]:
if False:
from unsloth import FastLanguageModel
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "granite_lora", # YOUR MODEL YOU USED FOR TRAINING
max_seq_length = 2048,
load_in_4bit = True,
)In [ ]:
# Merge to 16bit
if False:
model.save_pretrained_merged("granite_finetune_16bit", tokenizer, save_method = "merged_16bit",)
if False: # Pushing to HF Hub
model.push_to_hub_merged("HF_USERNAME/granite_finetune_16bit", tokenizer, save_method = "merged_16bit", token = "YOUR_HF_TOKEN")
# Merge to 4bit
if False:
model.save_pretrained_merged("granite_finetune_4bit", tokenizer, save_method = "merged_4bit",)
if False: # Pushing to HF Hub
model.push_to_hub_merged("HF_USERNAME/granite_finetune_4bit", tokenizer, save_method = "merged_4bit", token = "YOUR_HF_TOKEN")
# Just LoRA adapters
if False:
model.save_pretrained("granite_lora")
tokenizer.save_pretrained("granite_lora")
if False: # Pushing to HF Hub
model.push_to_hub("HF_USERNAME/granite_lora", token = "YOUR_HF_TOKEN")
tokenizer.push_to_hub("HF_USERNAME/granite_lora", token = "YOUR_HF_TOKEN")In [ ]:
# Save to 8bit Q8_0
if False:
model.save_pretrained_gguf("granite_finetune", tokenizer,)
# Remember to go to https://huggingface.co/settings/tokens for a token!
# And change hf to your username!
if False:
model.push_to_hub_gguf("HF_USERNAME/granite_finetune", tokenizer, token = "YOUR_HF_TOKEN")
# Save to 16bit GGUF
if False:
model.save_pretrained_gguf("granite_finetune", tokenizer, quantization_method = "f16")
if False: # Pushing to HF Hub
model.push_to_hub_gguf("HF_USERNAME/granite_finetune", tokenizer, quantization_method = "f16", token = "YOUR_HF_TOKEN")
# Save to q4_k_m GGUF
if False:
model.save_pretrained_gguf("granite_finetune", tokenizer, quantization_method = "q4_k_m")
if False: # Pushing to HF Hub
model.push_to_hub_gguf("HF_USERNAME/granite_finetune", tokenizer, quantization_method = "q4_k_m", token = "YOUR_HF_TOKEN")
# Save to multiple GGUF options - much faster if you want multiple!
if False:
model.push_to_hub_gguf(
"HF_USERNAME/granite_finetune", # Change hf to your username!
tokenizer,
quantization_method = ["q4_k_m", "q8_0", "q5_k_m",],
token = "YOUR_HF_TOKEN", # Get a token at https://huggingface.co/settings/tokens
)Now, use the granite_finetune.Q8_0.gguf file or granite_finetune.Q4_K_M.gguf file in llama.cpp.
And we're done! If you have any questions on Unsloth, we have a Discord channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!
Some other resources:
- Train your own reasoning model - Llama GRPO notebook Free Colab
- Saving finetunes to Ollama. Free notebook
- Llama 3.2 Vision finetuning - Radiography use case. Free Colab
- See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our documentation!
This notebook and all Unsloth notebooks are licensed LGPL-3.0.