mirror of
https://github.com/furyhawk/home_stack.git
synced 2026-07-21 02:06:47 +00:00
Add Unsloth notebook for fast finetuning with Llama and Qwen models
- Introduced a new Jupyter notebook `unsloth_nb.ipynb` for utilizing the Unsloth library. - Implemented code to load and patch FastLanguageModel and FastQwen2Model for improved training speed. - Configured models to support 4-bit quantization for reduced memory usage. - Included detailed logging for model loading and system specifications.
This commit is contained in:
@@ -5,3 +5,7 @@ node_modules/
|
||||
/blob-report/
|
||||
/playwright/.cache/
|
||||
frontend/.env
|
||||
.mypy_cache/
|
||||
# Ignore Python cache files
|
||||
__pycache__/
|
||||
unsloth_compiled_cache/
|
||||
|
||||
@@ -21,6 +21,11 @@ dependencies = [
|
||||
"pydantic-settings<3.0.0,>=2.2.1",
|
||||
"sentry-sdk[fastapi]<2.0.0,>=1.40.6",
|
||||
"pyjwt<3.0.0,>=2.8.0",
|
||||
"unsloth>=2025.5.9",
|
||||
"ipykernel>=6.29.5",
|
||||
"ipywidgets>=8.1.7",
|
||||
"protobuf==3.20.3",
|
||||
"termcolor>=3.1.0",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Start Llama.cpp service using Podman with GPU support
|
||||
podman run -d --name llamacpp-server -p 8000:8000 --gpus all -v $HOME/models:/models localhost/local/llama.cpp:full-cuda -s -m /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf --port 8000 --host 0.0.0.0 -n 512 --n-gpu-layers 1
|
||||
Generated
+2374
-12
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,159 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "536c8892",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/user/projects/ai_stack/backend/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🦥 Unsloth Zoo will now patch everything to make training faster!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from unsloth import FastLanguageModel\n",
|
||||
"import torch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "07d7e4a8",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"==((====))== Unsloth 2025.5.9: Fast Llama patching. Transformers: 4.52.4.\n",
|
||||
" \\\\ /| NVIDIA RTX 3500 Ada Generation Laptop GPU. Num GPUs = 1. Max memory: 11.607 GB. Platform: Linux.\n",
|
||||
"O^O/ \\_/ \\ Torch: 2.7.0+cu126. CUDA: 8.9. CUDA Toolkit: 12.6. Triton: 3.3.0\n",
|
||||
"\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.30. FA2 = False]\n",
|
||||
" \"-____-\" Free license: http://github.com/unslothai/unsloth\n",
|
||||
"Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
|
||||
" model_name = \"unsloth/llama-3.1-8b-bnb-4bit\", # Well-tested model\n",
|
||||
" max_seq_length = 2048, # Context length - can be longer, but uses more memory\n",
|
||||
" load_in_4bit = True, # 4bit uses much less memory\n",
|
||||
" load_in_8bit = False, # A bit more accurate, uses 2x memory\n",
|
||||
" full_finetuning = False, # We have full finetuning now!\n",
|
||||
" # token = \"hf_...\", # use one if using gated models\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "ae5430bd",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
|
||||
"🦥 Unsloth Zoo will now patch everything to make training faster!\n",
|
||||
"==((====))== Unsloth 2025.5.9: Fast Qwen2 patching. Transformers: 4.52.4.\n",
|
||||
" \\\\ /| NVIDIA RTX 3500 Ada Generation Laptop GPU. Num GPUs = 1. Max memory: 11.607 GB. Platform: Linux.\n",
|
||||
"O^O/ \\_/ \\ Torch: 2.7.0+cu126. CUDA: 8.9. CUDA Toolkit: 12.6. Triton: 3.3.0\n",
|
||||
"\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.30. FA2 = False]\n",
|
||||
" \"-____-\" Free license: http://github.com/unslothai/unsloth\n",
|
||||
"Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from unsloth import FastQwen2Model\n",
|
||||
"import torch\n",
|
||||
"\n",
|
||||
"max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n",
|
||||
"dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n",
|
||||
"load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.\n",
|
||||
"\n",
|
||||
"# 4bit pre quantized models we support for 4x faster downloading + no OOMs.\n",
|
||||
"fourbit_models = [\n",
|
||||
" \"unsloth/Meta-Llama-3.1-8B-bnb-4bit\", # Llama-3.1 2x faster\n",
|
||||
" \"unsloth/Meta-Llama-3.1-70B-bnb-4bit\",\n",
|
||||
" \"unsloth/Mistral-Small-Instruct-2409\", # Mistral 22b 2x faster!\n",
|
||||
" \"unsloth/mistral-7b-instruct-v0.3-bnb-4bit\",\n",
|
||||
" \"unsloth/Phi-3.5-mini-instruct\", # Phi-3.5 2x faster!\n",
|
||||
" \"unsloth/Phi-3-medium-4k-instruct\",\n",
|
||||
" \"unsloth/gemma-2-27b-bnb-4bit\", # Gemma 2x faster!\n",
|
||||
"\n",
|
||||
" \"unsloth/Llama-3.2-1B-bnb-4bit\", # NEW! Llama 3.2 models\n",
|
||||
" \"unsloth/Llama-3.2-1B-Instruct-bnb-4bit\",\n",
|
||||
" \"unsloth/Llama-3.2-3B-Instruct-bnb-4bit\",\n",
|
||||
"] # More models at https://huggingface.co/unsloth\n",
|
||||
"\n",
|
||||
"qwen_models = [\n",
|
||||
" \"unsloth/Qwen2.5-Coder-32B-Instruct\", # Qwen 2.5 Coder 2x faster\n",
|
||||
" \"unsloth/Qwen2.5-Coder-7B\",\n",
|
||||
" \"unsloth/Qwen2.5-14B-Instruct\", # 14B fits in a 16GB card\n",
|
||||
" \"unsloth/Qwen2.5-7B\",\n",
|
||||
" \"unsloth/Qwen2.5-72B-Instruct\", # 72B fits in a 48GB card\n",
|
||||
"] # More models at https://huggingface.co/unsloth\n",
|
||||
"\n",
|
||||
"model, tokenizer = FastQwen2Model.from_pretrained(\n",
|
||||
" model_name=\"unsloth/Qwen2.5-Coder-1.5B-Instruct\",\n",
|
||||
" max_seq_length=None,\n",
|
||||
" dtype=None,\n",
|
||||
" load_in_4bit=False,\n",
|
||||
" fix_tokenizer=False\n",
|
||||
" # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a8dfe1a4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.16"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user