From a8dad3debe79b27f009f1fe7e18fe17fef338053 Mon Sep 17 00:00:00 2001 From: furyhawk Date: Sun, 15 Jun 2025 08:29:42 +0800 Subject: [PATCH] fix: Update model file references and enhance Dockerfile for LlamaCPP setup --- ai_stack/llamacpp/Dockerfile | 9 ++++++++- ai_stack/llamacpp/README.md | 2 +- ai_stack/llamacpp/docker-compose.yml | 12 ++++-------- ai_stack/ollama/docker-compose.yml | 1 + makefile | 7 ++++++- scripts/llamacpp-gpu.sh | 2 +- scripts/pull-deepseek-llamacpp.sh | 2 +- scripts/pull-deepseek-model.sh | 2 +- 8 files changed, 23 insertions(+), 14 deletions(-) diff --git a/ai_stack/llamacpp/Dockerfile b/ai_stack/llamacpp/Dockerfile index c92f180..eb1195d 100644 --- a/ai_stack/llamacpp/Dockerfile +++ b/ai_stack/llamacpp/Dockerfile @@ -7,6 +7,8 @@ RUN apt-get update && apt-get install -y \ g++ \ wget \ curl \ + libcurl4-openssl-dev \ + pkg-config \ && rm -rf /var/lib/apt/lists/* # Clone llama.cpp @@ -16,6 +18,11 @@ RUN git clone https://github.com/ggerganov/llama.cpp.git WORKDIR /llama.cpp RUN mkdir build && cd build && cmake .. -DLLAMA_SERVER=ON && make -j$(nproc) +# Copy the built executable to a standard location +RUN cp /llama.cpp/build/bin/llama-server /usr/local/bin/llama-server || \ + cp /llama.cpp/build/llama-server /usr/local/bin/llama-server || \ + find /llama.cpp -name "llama-server" -type f -executable -exec cp {} /usr/local/bin/llama-server \; + # Create models directory RUN mkdir -p /models @@ -27,4 +34,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD curl -f http://localhost:8080/health || exit 1 # Default command - will be overridden by docker-compose -CMD ["/llama.cpp/llama-server", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file +CMD ["llama-server", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/ai_stack/llamacpp/README.md b/ai_stack/llamacpp/README.md index 36e7eba..5b63e54 100644 --- a/ai_stack/llamacpp/README.md +++ b/ai_stack/llamacpp/README.md @@ -173,7 +173,7 @@ ls -la ai_stack/llamacpp/models/ # Test with minimal configuration docker run --rm -v $(pwd)/ai_stack/llamacpp/models:/models \ llamacpp /llama.cpp/llama-server \ - --model /models/DeepSeek-R1-0528-Qwen3-8B-Q4_K_XL.gguf \ + --model /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf \ --host 0.0.0.0 --port 8080 --ctx-size 2048 ``` diff --git a/ai_stack/llamacpp/docker-compose.yml b/ai_stack/llamacpp/docker-compose.yml index 1d4f618..7f2e162 100644 --- a/ai_stack/llamacpp/docker-compose.yml +++ b/ai_stack/llamacpp/docker-compose.yml @@ -2,6 +2,7 @@ version: '3.8' services: llama-cpp-server: build: . + image: localhost/llama-cpp-server:latest ports: - "8080:8080" # Map host port 8080 to container port 8080 volumes: @@ -12,21 +13,16 @@ services: - LLAMA_ARG_PORT=8080 restart: unless-stopped command: [ - "/llama.cpp/llama-server", - "--model", "/models/DeepSeek-R1-0528-Qwen3-8B-Q4_K_XL.gguf", + "llama-server", + "--model", "/models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf", "--host", "0.0.0.0", "--port", "8080", "--ctx-size", "32768", - "--n-predict", "2048", "--threads", "4", "--batch-size", "512", "--cont-batching", "--parallel", "2", - "--flash-attn", - "--temperature", "0.7", - "--top-p", "0.9", - "--top-k", "40", - "--repeat-penalty", "1.1" + "--flash-attn" ] # Uncomment for GPU support # deploy: diff --git a/ai_stack/ollama/docker-compose.yml b/ai_stack/ollama/docker-compose.yml index 136f0b2..ac549d7 100644 --- a/ai_stack/ollama/docker-compose.yml +++ b/ai_stack/ollama/docker-compose.yml @@ -2,6 +2,7 @@ version: '3.8' services: ollama: build: . + image: localhost/ollama-server:latest ports: - "11434:11434" # Map host port 11434 to container port 11434 (Ollama default) volumes: diff --git a/makefile b/makefile index 711e592..ea4cad2 100644 --- a/makefile +++ b/makefile @@ -12,7 +12,7 @@ include $(ENV_FILE) export # Targets -.PHONY: up down restart logs build reset network pull-deepseek-model ollama-up ollama-down ollama-logs llamacpp-up llamacpp-down llamacpp-logs pull-deepseek-llamacpp llamacpp-gpu ai-launcher +.PHONY: up down restart logs build reset network pull-deepseek-model ollama-up ollama-down ollama-logs llamacpp-up llamacpp-down llamacpp-logs llamacpp-build pull-deepseek-llamacpp llamacpp-gpu ai-launcher network: @podman network exists traefik-public || podman network create traefik-public @@ -103,6 +103,7 @@ help: @echo " make setup-ollama Start Ollama and automatically pull DeepSeek model" @echo "" @echo "LlamaCPP-specific targets:" + @echo " make llamacpp-build Build LlamaCPP container with no cache" @echo " make llamacpp-up Start only the LlamaCPP container" @echo " make llamacpp-down Stop only the LlamaCPP container" @echo " make llamacpp-logs View LlamaCPP container logs" @@ -156,6 +157,10 @@ setup-ollama: ollama-up @$(MAKE) pull-deepseek-model # LlamaCPP-specific targets +llamacpp-build: + cd ai_stack/llamacpp && DOCKER_BUILDKIT=0 podman compose build --no-cache + @echo "LlamaCPP container built with no cache. Use 'make llamacpp-up' to start it." + llamacpp-up: cd ai_stack/llamacpp && podman compose up -d @echo "LlamaCPP container started. Use 'make pull-deepseek-llamacpp' to download the DeepSeek model." diff --git a/scripts/llamacpp-gpu.sh b/scripts/llamacpp-gpu.sh index 0c78ec1..e01ffab 100755 --- a/scripts/llamacpp-gpu.sh +++ b/scripts/llamacpp-gpu.sh @@ -6,7 +6,7 @@ set -e # Configuration -MODEL_FILE="DeepSeek-R1-0528-Qwen3-8B-Q4_K_XL.gguf" +MODEL_FILE="DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf" MODELS_DIR="./ai_stack/llamacpp/models" CONTAINER_NAME="llama-cpp-server-gpu" HOST_PORT="8080" diff --git a/scripts/pull-deepseek-llamacpp.sh b/scripts/pull-deepseek-llamacpp.sh index 70b6540..26e5de8 100755 --- a/scripts/pull-deepseek-llamacpp.sh +++ b/scripts/pull-deepseek-llamacpp.sh @@ -7,7 +7,7 @@ set -e # Configuration MODEL_REPO="unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF" -MODEL_FILE="DeepSeek-R1-0528-Qwen3-8B-Q4_K_XL.gguf" +MODEL_FILE="DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf" MODELS_DIR="./ai_stack/llamacpp/models" LLAMACPP_HOST="http://localhost:8080" CONTAINER_NAME="llama-cpp-server" diff --git a/scripts/pull-deepseek-model.sh b/scripts/pull-deepseek-model.sh index 1329824..fbeedd3 100755 --- a/scripts/pull-deepseek-model.sh +++ b/scripts/pull-deepseek-model.sh @@ -7,7 +7,7 @@ set -e # Configuration MODEL_REPO="unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF" -MODEL_FILE="DeepSeek-R1-0528-Qwen3-8B-Q4_K_XL.gguf" +MODEL_FILE="DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf" OLLAMA_HOST="http://localhost:11434" MODEL_NAME="deepseek-r1-qwen3-8b" MODELS_DIR="./ai_stack/ollama/models"