fix: Update model file references and enhance Dockerfile for LlamaCPP setup

This commit is contained in:
2025-06-15 08:29:42 +08:00
parent e9dc288cef
commit a8dad3debe
8 changed files with 23 additions and 14 deletions
+8 -1
View File
@@ -7,6 +7,8 @@ RUN apt-get update && apt-get install -y \
g++ \
wget \
curl \
libcurl4-openssl-dev \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
# Clone llama.cpp
@@ -16,6 +18,11 @@ RUN git clone https://github.com/ggerganov/llama.cpp.git
WORKDIR /llama.cpp
RUN mkdir build && cd build && cmake .. -DLLAMA_SERVER=ON && make -j$(nproc)
# Copy the built executable to a standard location
RUN cp /llama.cpp/build/bin/llama-server /usr/local/bin/llama-server || \
cp /llama.cpp/build/llama-server /usr/local/bin/llama-server || \
find /llama.cpp -name "llama-server" -type f -executable -exec cp {} /usr/local/bin/llama-server \;
# Create models directory
RUN mkdir -p /models
@@ -27,4 +34,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8080/health || exit 1
# Default command - will be overridden by docker-compose
CMD ["/llama.cpp/llama-server", "--host", "0.0.0.0", "--port", "8080"]
CMD ["llama-server", "--host", "0.0.0.0", "--port", "8080"]
+1 -1
View File
@@ -173,7 +173,7 @@ ls -la ai_stack/llamacpp/models/
# Test with minimal configuration
docker run --rm -v $(pwd)/ai_stack/llamacpp/models:/models \
llamacpp /llama.cpp/llama-server \
--model /models/DeepSeek-R1-0528-Qwen3-8B-Q4_K_XL.gguf \
--model /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf \
--host 0.0.0.0 --port 8080 --ctx-size 2048
```
+4 -8
View File
@@ -2,6 +2,7 @@ version: '3.8'
services:
llama-cpp-server:
build: .
image: localhost/llama-cpp-server:latest
ports:
- "8080:8080" # Map host port 8080 to container port 8080
volumes:
@@ -12,21 +13,16 @@ services:
- LLAMA_ARG_PORT=8080
restart: unless-stopped
command: [
"/llama.cpp/llama-server",
"--model", "/models/DeepSeek-R1-0528-Qwen3-8B-Q4_K_XL.gguf",
"llama-server",
"--model", "/models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf",
"--host", "0.0.0.0",
"--port", "8080",
"--ctx-size", "32768",
"--n-predict", "2048",
"--threads", "4",
"--batch-size", "512",
"--cont-batching",
"--parallel", "2",
"--flash-attn",
"--temperature", "0.7",
"--top-p", "0.9",
"--top-k", "40",
"--repeat-penalty", "1.1"
"--flash-attn"
]
# Uncomment for GPU support
# deploy:
+1
View File
@@ -2,6 +2,7 @@ version: '3.8'
services:
ollama:
build: .
image: localhost/ollama-server:latest
ports:
- "11434:11434" # Map host port 11434 to container port 11434 (Ollama default)
volumes: