{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d4ed83af", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "โœ… Creating LlamaCPP API Wrapper\n", "This provides a clean interface to http://ollama.lan:8080/\n" ] } ], "source": [ "import requests\n", "import json\n", "from typing import Dict, Any, List, Optional, Iterator\n", "from dataclasses import dataclass\n", "\n", "print(\"โœ… Creating LlamaCPP API Wrapper\")\n", "print(\"This provides a clean interface to http://ollama.lan:8080/\")" ] }, { "cell_type": "code", "execution_count": 2, "id": "ecedf21c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Initializing LlamaCPP API Wrapper ===\n", "โœ… Discovered model: /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n", "โœ… Successfully connected to LlamaCPP API\n", "๐Ÿ”— Endpoint: http://ollama.lan:8080\n", "๐Ÿค– Auto-discovered model: /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n" ] } ], "source": [ "import os\n", "import requests\n", "import json\n", "from typing import Optional, Dict, Any, List, Iterator\n", "\n", "class LlamaCPPAPIWrapper:\n", " \"\"\"\n", " A clean wrapper for the LlamaCPP API at http://ollama.lan:8080/\n", " Provides a simple, consistent interface similar to OpenAI's API\n", " \"\"\"\n", " \n", " def __init__(self, base_url: str = \"http://ollama.lan:8080\", timeout: int = 300):\n", " self.base_url = base_url.rstrip(\"/\")\n", " self.timeout = timeout\n", " self.model_name = None\n", " self._discover_model()\n", " \n", " def _discover_model(self):\n", " \"\"\"Automatically discover available model\"\"\"\n", " try:\n", " response = self._make_request(\"/v1/models\", method=\"GET\")\n", " if response.status_code == 200:\n", " models_data = response.json()\n", " if 'data' in models_data and len(models_data['data']) > 0:\n", " self.model_name = models_data['data'][0]['id']\n", " print(f\"โœ… Discovered model: {self.model_name}\")\n", " except Exception as e:\n", " print(f\"โš ๏ธ Could not auto-discover model: {e}\")\n", " \n", " def _make_request(self, endpoint: str, data: Optional[Dict] = None, method: str = \"GET\") -> requests.Response:\n", " \"\"\"Helper method for making HTTP requests\"\"\"\n", " url = f\"{self.base_url}{endpoint}\"\n", " headers = {\"Content-Type\": \"application/json\"}\n", " \n", " try:\n", " if method.upper() == \"GET\":\n", " return requests.get(url, timeout=self.timeout)\n", " elif method.upper() == \"POST\":\n", " return requests.post(url, json=data, headers=headers, timeout=self.timeout)\n", " else:\n", " raise ValueError(f\"Unsupported method: {method}\")\n", " except requests.exceptions.RequestException as e:\n", " raise ConnectionError(f\"Failed to connect to {url}: {e}\")\n", " \n", " def health_check(self) -> bool:\n", " \"\"\"Check if the API server is healthy\"\"\"\n", " try:\n", " response = self._make_request(\"/health\")\n", " return response.status_code == 200\n", " except:\n", " return False\n", " \n", " def list_models(self) -> List[Dict]:\n", " \"\"\"List available models\"\"\"\n", " try:\n", " response = self._make_request(\"/v1/models\")\n", " if response.status_code == 200:\n", " return response.json().get('data', [])\n", " return []\n", " except Exception as e:\n", " print(f\"Error listing models: {e}\")\n", " return []\n", " \n", " def complete(\n", " self,\n", " prompt: str,\n", " model: Optional[str] = None,\n", " max_tokens: int = 100,\n", " temperature: float = 0.7,\n", " top_p: float = 0.95,\n", " stop: Optional[List[str]] = None,\n", " stream: bool = False\n", " ) -> Dict[str, Any]:\n", " \"\"\"\n", " Generate text completion\n", " \n", " Args:\n", " prompt: Input text prompt\n", " model: Model name (uses auto-discovered if None)\n", " max_tokens: Maximum tokens to generate\n", " temperature: Sampling temperature (0.0 to 2.0)\n", " top_p: Nucleus sampling parameter\n", " stop: Stop sequences\n", " stream: Whether to stream the response\n", " \"\"\"\n", " model = model or self.model_name\n", " if not model:\n", " raise ValueError(\"No model specified and none auto-discovered\")\n", " \n", " data = {\n", " \"model\": model,\n", " \"prompt\": prompt,\n", " \"max_tokens\": max_tokens,\n", " \"temperature\": temperature,\n", " \"top_p\": top_p,\n", " \"stream\": stream\n", " }\n", " \n", " if stop:\n", " data[\"stop\"] = stop\n", " \n", " response = self._make_request(\"/v1/completions\", data, \"POST\")\n", " \n", " if response.status_code == 200:\n", " return response.json()\n", " else:\n", " raise RuntimeError(f\"API error: {response.status_code} - {response.text}\")\n", " \n", " def chat(\n", " self,\n", " messages: List[Dict[str, str]],\n", " model: Optional[str] = None,\n", " max_tokens: int = 100,\n", " temperature: float = 0.7,\n", " top_p: float = 0.95,\n", " stream: bool = False\n", " ) -> Dict[str, Any]:\n", " \"\"\"\n", " Generate chat completion\n", " \n", " Args:\n", " messages: List of message dicts with 'role' and 'content'\n", " model: Model name (uses auto-discovered if None)\n", " max_tokens: Maximum tokens to generate\n", " temperature: Sampling temperature\n", " top_p: Nucleus sampling parameter\n", " stream: Whether to stream the response\n", " \"\"\"\n", " model = model or self.model_name\n", " if not model:\n", " raise ValueError(\"No model specified and none auto-discovered\")\n", " \n", " data = {\n", " \"model\": model,\n", " \"messages\": messages,\n", " \"max_tokens\": max_tokens,\n", " \"temperature\": temperature,\n", " \"top_p\": top_p,\n", " \"stream\": stream\n", " }\n", " \n", " response = self._make_request(\"/v1/chat/completions\", data, \"POST\")\n", " \n", " if response.status_code == 200:\n", " return response.json()\n", " else:\n", " raise RuntimeError(f\"API error: {response.status_code} - {response.text}\")\n", " \n", " def stream_complete(\n", " self,\n", " prompt: str,\n", " model: Optional[str] = None,\n", " max_tokens: int = 100,\n", " temperature: float = 0.7,\n", " **kwargs\n", " ) -> Iterator[str]:\n", " \"\"\"\n", " Stream text completion tokens\n", " \n", " Yields individual tokens as they're generated\n", " \"\"\"\n", " model = model or self.model_name\n", " if not model:\n", " raise ValueError(\"No model specified and none auto-discovered\")\n", " \n", " data = {\n", " \"model\": model,\n", " \"prompt\": prompt,\n", " \"max_tokens\": max_tokens,\n", " \"temperature\": temperature,\n", " \"stream\": True,\n", " **kwargs\n", " }\n", " \n", " response = self._make_request(\"/v1/completions\", data, \"POST\")\n", " \n", " if response.status_code != 200:\n", " raise RuntimeError(f\"API error: {response.status_code} - {response.text}\")\n", " \n", " # Process streaming response\n", " for line in response.iter_lines():\n", " if line:\n", " line_text = line.decode('utf-8')\n", " if line_text.startswith('data: '):\n", " data_part = line_text[6:]\n", " if data_part.strip() == '[DONE]':\n", " break\n", " try:\n", " chunk = json.loads(data_part)\n", " if 'choices' in chunk and len(chunk['choices']) > 0:\n", " content = chunk['choices'][0].get('text', '')\n", " if content:\n", " yield content\n", " except json.JSONDecodeError:\n", " continue\n", " \n", " def stream_chat(\n", " self,\n", " messages: List[Dict[str, str]],\n", " model: Optional[str] = None,\n", " max_tokens: int = 100,\n", " temperature: float = 0.7,\n", " **kwargs\n", " ) -> Iterator[str]:\n", " \"\"\"\n", " Stream chat completion tokens\n", " \n", " Yields individual tokens as they're generated\n", " \"\"\"\n", " model = model or self.model_name\n", " if not model:\n", " raise ValueError(\"No model specified and none auto-discovered\")\n", " \n", " data = {\n", " \"model\": model,\n", " \"messages\": messages,\n", " \"max_tokens\": max_tokens,\n", " \"temperature\": temperature,\n", " \"stream\": True,\n", " **kwargs\n", " }\n", " \n", " response = self._make_request(\"/v1/chat/completions\", data, \"POST\")\n", " \n", " if response.status_code != 200:\n", " raise RuntimeError(f\"API error: {response.status_code} - {response.text}\")\n", " \n", " # Process streaming response\n", " for line in response.iter_lines():\n", " if line:\n", " line_text = line.decode('utf-8')\n", " if line_text.startswith('data: '):\n", " data_part = line_text[6:]\n", " if data_part.strip() == '[DONE]':\n", " break\n", " try:\n", " chunk = json.loads(data_part)\n", " if 'choices' in chunk and len(chunk['choices']) > 0:\n", " delta = chunk['choices'][0].get('delta', {})\n", " content = delta.get('content', '')\n", " if content:\n", " yield content\n", " except json.JSONDecodeError:\n", " continue\n", "\n", "# Initialize the wrapper\n", "print(\"=== Initializing LlamaCPP API Wrapper ===\")\n", "llm_api = LlamaCPPAPIWrapper()\n", "\n", "# Test connection\n", "if llm_api.health_check():\n", " print(\"โœ… Successfully connected to LlamaCPP API\")\n", " print(f\"๐Ÿ”— Endpoint: {llm_api.base_url}\")\n", " if llm_api.model_name:\n", " print(f\"๐Ÿค– Auto-discovered model: {llm_api.model_name}\")\n", "else:\n", " print(\"โŒ Failed to connect to LlamaCPP API\")\n", " print(\"Please check that the server is running at http://ollama.lan:8080/\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "a308bccd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Example 1: Health Check & Model Discovery ===\n", "API Health: โœ… Healthy\n", "Available models: 1\n", " 1. /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n", " Type: model\n", "\n", "๐ŸŽฏ Using model: /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n" ] } ], "source": [ "# Example 1: API Health Check and Model Discovery\n", "print(\"=== Example 1: Health Check & Model Discovery ===\")\n", "\n", "try:\n", " # Check API health\n", " is_healthy = llm_api.health_check()\n", " print(f\"API Health: {'โœ… Healthy' if is_healthy else 'โŒ Unhealthy'}\")\n", " \n", " # List available models\n", " models = llm_api.list_models()\n", " print(f\"Available models: {len(models)}\")\n", " \n", " for i, model in enumerate(models[:3]): # Show first 3 models\n", " print(f\" {i+1}. {model.get('id', 'Unknown')}\")\n", " if 'object' in model:\n", " print(f\" Type: {model['object']}\")\n", " \n", " if llm_api.model_name:\n", " print(f\"\\n๐ŸŽฏ Using model: {llm_api.model_name}\")\n", " else:\n", " print(\"\\nโš ๏ธ No model auto-discovered\")\n", " \n", "except Exception as e:\n", " print(f\"โŒ Error: {e}\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "23ea3a7b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Example 2: Text Completion ===\n", "Prompt: 'The capital of France is'\n", "โœ… Completion: 'The capital of France is a city known for its art, culture, and cuisine'\n", "๐Ÿ“Š Tokens - Prompt: 5, Completion: 12, Total: 17\n", "โœ… Completion: 'The capital of France is a city known for its art, culture, and cuisine'\n", "๐Ÿ“Š Tokens - Prompt: 5, Completion: 12, Total: 17\n" ] } ], "source": [ "# Example 2: Simple Text Completion using Wrapper\n", "print(\"=== Example 2: Text Completion ===\")\n", "\n", "try:\n", " prompt = \"The capital of France is\"\n", " print(f\"Prompt: '{prompt}'\")\n", " \n", " # Use the wrapper's complete method\n", " result = llm_api.complete(\n", " prompt=prompt,\n", " max_tokens=50,\n", " temperature=0.7,\n", " stop=[\"\\n\", \".\", \"!\"]\n", " )\n", " \n", " if 'choices' in result and len(result['choices']) > 0:\n", " completion = result['choices'][0]['text']\n", " print(f\"โœ… Completion: '{prompt}{completion}'\")\n", " \n", " if 'usage' in result:\n", " usage = result['usage']\n", " print(f\"๐Ÿ“Š Tokens - Prompt: {usage.get('prompt_tokens', 'N/A')}, \"\n", " f\"Completion: {usage.get('completion_tokens', 'N/A')}, \"\n", " f\"Total: {usage.get('total_tokens', 'N/A')}\")\n", " else:\n", " print(\"โŒ No completion returned\")\n", " \n", "except Exception as e:\n", " print(f\"โŒ Error during completion: {e}\")" ] }, { "cell_type": "code", "execution_count": 5, "id": "53703142", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Example 3: Chat Completion ===\n", "๐Ÿ’ฌ Chat Messages:\n", " System: You are a helpful assistant that gives concise answers.\n", " User: Explain quantum computing in simple terms.\n", "\n", "Assistant: \n", "Okay, the user asked for a simple explanation of quantum computing. Let me start by breaking down their request. They probably want a clear, non-technical overview without too much jargon. But maybe they're curious because they heard about it in the news or from someone, and they need a basic understanding to follow along or grasp why it's important.\n", "\n", "First, I should compare classical computing to quantum computing. People know computers use bits, so starting with that makes sense. But the user\n", "\n", "๐Ÿ“Š Usage: {'completion_tokens': 100, 'prompt_tokens': 20, 'total_tokens': 120}\n", "\n", "Okay, the user asked for a simple explanation of quantum computing. Let me start by breaking down their request. They probably want a clear, non-technical overview without too much jargon. But maybe they're curious because they heard about it in the news or from someone, and they need a basic understanding to follow along or grasp why it's important.\n", "\n", "First, I should compare classical computing to quantum computing. People know computers use bits, so starting with that makes sense. But the user\n", "\n", "๐Ÿ“Š Usage: {'completion_tokens': 100, 'prompt_tokens': 20, 'total_tokens': 120}\n" ] } ], "source": [ "# Example 3: Chat Completion using Wrapper\n", "print(\"=== Example 3: Chat Completion ===\")\n", "\n", "try:\n", " messages = [\n", " {\"role\": \"system\", \"content\": \"You are a helpful assistant that gives concise answers.\"},\n", " {\"role\": \"user\", \"content\": \"Explain quantum computing in simple terms.\"}\n", " ]\n", " \n", " print(\"๐Ÿ’ฌ Chat Messages:\")\n", " for msg in messages:\n", " print(f\" {msg['role'].title()}: {msg['content']}\")\n", " \n", " print(\"\\nAssistant: \", end=\"\", flush=True)\n", " \n", " # Use the wrapper's chat method\n", " result = llm_api.chat(\n", " messages=messages,\n", " max_tokens=100,\n", " temperature=0.7\n", " )\n", " \n", " if 'choices' in result and len(result['choices']) > 0:\n", " response_message = result['choices'][0]['message']['content']\n", " print(response_message)\n", " \n", " if 'usage' in result:\n", " usage = result['usage']\n", " print(f\"\\n๐Ÿ“Š Usage: {usage}\")\n", " else:\n", " print(\"โŒ No response returned\")\n", " \n", "except Exception as e:\n", " print(f\"โŒ Error during chat completion: {e}\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "994492b6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Example 4: Streaming Completion ===\n", "Prompt: Write a short poem about artificial intelligence:\n", "Response: 5 lines.\n", "Here's a short poem about artificial intelligence with exactly five lines:\n", "\n", "In circuits of cold light, \n", "Machines learn 5 lines.\n", "Here's a short poem about artificial intelligence with exactly five lines:\n", "\n", "In circuits of cold light, \n", "Machines learn and dream bright, \n", "Data flows in endless streams, \n", "Answers from the AI streams, \n", "A silent, thinking streams.\n", "Hmm, the user asked for a and dream bright, \n", "Data flows in endless streams, \n", "Answers from the AI streams, \n", "A silent, thinking streams.\n", "Hmm, the user asked for a short poem about artificial intelligence with exactly five lines. They're probably looking short poem about artificial intelligence with exactly five lines. They're probably looking for something creative and concise, maybe\n", "\n", "โœ… Streaming completed!\n", "๐Ÿ“ Total response: 396 characters\n", " for something creative and concise, maybe\n", "\n", "โœ… Streaming completed!\n", "๐Ÿ“ Total response: 396 characters\n" ] } ], "source": [ "# Example 4: Streaming Text Completion using Wrapper\n", "print(\"=== Example 4: Streaming Completion ===\")\n", "\n", "try:\n", " prompt = \"Write a short poem about artificial intelligence:\"\n", " print(f\"Prompt: {prompt}\")\n", " print(\"Response: \", end=\"\", flush=True)\n", " \n", " # Use the wrapper's streaming method\n", " full_response = \"\"\n", " for token in llm_api.stream_complete(\n", " prompt=prompt,\n", " max_tokens=80,\n", " temperature=0.8\n", " ):\n", " full_response += token\n", " print(token, end=\"\", flush=True)\n", " \n", " print(f\"\\n\\nโœ… Streaming completed!\")\n", " print(f\"๐Ÿ“ Total response: {len(full_response)} characters\")\n", " \n", "except Exception as e:\n", " print(f\"โŒ Error during streaming: {e}\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "94dc6845", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Example 5: Streaming Chat ===\n", "๐Ÿ’ฌ Starting streaming chat...\n", "User: Tell me a short story about a robot discovering emotions.\n", "Assistant: \n", "Okay, user wants a short story about a robot discovering emotions. That's a pretty interesting prompt - it suggests they're interested in both sci-fi and emotional exploration. Maybe\n", "Okay, user wants a short story about a robot discovering emotions. That's a pretty interesting prompt - it suggests they're interested in both sci-fi and emotional exploration. Maybe they're a writer looking for inspiration, or just someone curious about how emotionless beings might experience feelings. \n", "\n", "Hmm, the challenge here is to make a robot's emotional awakening feel believable and touching they're a writer looking for inspiration, or just someone curious about how emotionless beings might experience feelings. \n", "\n", "Hmm, the challenge here is to make a robot's emotional awakening feel believable and touching. Can't just have it suddenly \"feel\". Can't just have it suddenly \"feel\" things - that would be too abrupt. Needs a gradual realization, with concrete examples. \n", "\n", "I should start with a robot that's clearly designed to be emotionless, maybe a service bot in a sterile environment. The discovery should happen through human interactions - that feels authentic. Perhaps an elderly user things - that would be too abrupt. Needs a gradual realization, with concrete examples. \n", "\n", "I should start with a robot that's clearly designed to be emotionless, maybe a service bot in a sterile environment. The discovery should happen through human interactions - that feels authentic. Perhaps an elderly user? Their simple requests could highlight how\n", "\n", "โœ… Streaming chat completed!\n", "๐Ÿ“ Response length: 797 characters\n", "? Their simple requests could highlight how\n", "\n", "โœ… Streaming chat completed!\n", "๐Ÿ“ Response length: 797 characters\n" ] } ], "source": [ "# Example 5: Streaming Chat Completion using Wrapper\n", "print(\"=== Example 5: Streaming Chat ===\")\n", "\n", "try:\n", " messages = [\n", " {\"role\": \"system\", \"content\": \"You are a creative writing assistant.\"},\n", " {\"role\": \"user\", \"content\": \"Tell me a short story about a robot discovering emotions.\"}\n", " ]\n", " \n", " print(\"๐Ÿ’ฌ Starting streaming chat...\")\n", " print(f\"User: {messages[1]['content']}\")\n", " print(\"Assistant: \", end=\"\", flush=True)\n", " \n", " # Use the wrapper's streaming chat method\n", " full_response = \"\"\n", " for token in llm_api.stream_chat(\n", " messages=messages,\n", " max_tokens=150,\n", " temperature=0.8\n", " ):\n", " full_response += token\n", " print(token, end=\"\", flush=True)\n", " \n", " print(f\"\\n\\nโœ… Streaming chat completed!\")\n", " print(f\"๐Ÿ“ Response length: {len(full_response)} characters\")\n", " \n", "except Exception as e:\n", " print(f\"โŒ Error during streaming chat: {e}\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "edc1ea4b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Example 6: Embeddings ===\n", "Input text: 'Hello, world! This is a test sentence for embeddings.'\n", "Testing embeddings endpoint...\n", "โŒ Embeddings not supported: 400 - {\"error\":{\"code\":400,\"message\":\"Pooling type 'none' is not OAI compatible. Please use a different pooling type\",\"type\":\"invalid_request_error\"}}\n" ] } ], "source": [ "# Example 6: Embeddings Test (if supported)\n", "print(\"=== Example 6: Embeddings ===\")\n", "\n", "try:\n", " # Test if embeddings endpoint is available\n", " embeddings_data = {\n", " \"model\": llm_api.model_name,\n", " \"input\": \"Hello, world! This is a test sentence for embeddings.\"\n", " }\n", " \n", " print(f\"Input text: '{embeddings_data['input']}'\")\n", " print(\"Testing embeddings endpoint...\")\n", " \n", " try:\n", " response = llm_api._make_request(\"/v1/embeddings\", embeddings_data, \"POST\")\n", " \n", " if response.status_code == 200:\n", " result = response.json()\n", " if 'data' in result and len(result['data']) > 0:\n", " embedding = result['data'][0]['embedding']\n", " print(f\"โœ… Embeddings generated successfully!\")\n", " print(f\"๐Ÿ“Š Embedding dimensions: {len(embedding)}\")\n", " print(f\"๐Ÿ”ข First 10 values: {embedding[:10]}\")\n", " if 'usage' in result:\n", " print(f\"๐Ÿ“ˆ Usage: {result['usage']}\")\n", " else:\n", " print(\"โŒ No embedding data in response\")\n", " else:\n", " print(f\"โŒ Embeddings not supported: {response.status_code} - {response.text}\")\n", " \n", " except Exception as embed_error:\n", " print(f\"โŒ Embeddings endpoint error: {embed_error}\")\n", " print(\"๐Ÿ’ก This model/server may not support embeddings\")\n", " \n", "except Exception as e:\n", " print(f\"โŒ Error: {e}\")" ] }, { "cell_type": "code", "execution_count": 9, "id": "6c23615c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "=== Example 7: Utility Functions ===\n", "๐Ÿงช Testing utility functions:\n", "\n", "1. Quick chat:\n", "Response: \n", "Hmm, \"What is the meaning of life?\" - that's a classic. The user is asking one of the most profound philosophical questions out there. \n", "\n", "First, I should acknowledge that this is a deeply personal question with no single \"correct\n", "\n", "2. Quick completion:\n", "Response: \n", "Hmm, \"What is the meaning of life?\" - that's a classic. The user is asking one of the most profound philosophical questions out there. \n", "\n", "First, I should acknowledge that this is a deeply personal question with no single \"correct\n", "\n", "2. Quick completion:\n", "Response: the one that compiles the fastest. โ€” This is a common misconception in the programming community. In reality, the performance of a compiled language like C\n", "\n", "3. API info:\n", " endpoint: http://ollama.lan:8080\n", " health: True\n", " model_count: 1\n", " current_model: /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n", " available_models: ['/models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf']\n", "\n", "=== Summary ===\n", "โœ… LlamaCPP API Wrapper Examples Complete!\n", "๐Ÿ”— API Endpoint: http://ollama.lan:8080\n", "๐Ÿค– Model: /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n", "\n", "๐Ÿ”ง Wrapper Features Demonstrated:\n", " - โœ… Automatic model discovery\n", " - โœ… Health checking\n", " - โœ… Text completion\n", " - โœ… Chat completion\n", " - โœ… Streaming completion\n", " - โœ… Streaming chat\n", " - โœ… Error handling\n", " - โœ… Clean, consistent API\n", "\n", "๐Ÿ’ก Wrapper Benefits:\n", " - Simple, intuitive interface\n", " - Automatic error handling\n", " - Type hints for better IDE support\n", " - Consistent parameter naming\n", " - Built-in streaming support\n", " - Connection management\n", "Response: the one that compiles the fastest. โ€” This is a common misconception in the programming community. In reality, the performance of a compiled language like C\n", "\n", "3. API info:\n", " endpoint: http://ollama.lan:8080\n", " health: True\n", " model_count: 1\n", " current_model: /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n", " available_models: ['/models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf']\n", "\n", "=== Summary ===\n", "โœ… LlamaCPP API Wrapper Examples Complete!\n", "๐Ÿ”— API Endpoint: http://ollama.lan:8080\n", "๐Ÿค– Model: /models/DeepSeek-R1-0528-Qwen3-8B-UD-Q4_K_XL.gguf\n", "\n", "๐Ÿ”ง Wrapper Features Demonstrated:\n", " - โœ… Automatic model discovery\n", " - โœ… Health checking\n", " - โœ… Text completion\n", " - โœ… Chat completion\n", " - โœ… Streaming completion\n", " - โœ… Streaming chat\n", " - โœ… Error handling\n", " - โœ… Clean, consistent API\n", "\n", "๐Ÿ’ก Wrapper Benefits:\n", " - Simple, intuitive interface\n", " - Automatic error handling\n", " - Type hints for better IDE support\n", " - Consistent parameter naming\n", " - Built-in streaming support\n", " - Connection management\n" ] } ], "source": [ "# Example 7: Utility Functions using the Wrapper\n", "print(\"=== Example 7: Utility Functions ===\")\n", "\n", "def quick_chat(prompt: str, max_tokens: int = 100, temperature: float = 0.7) -> str:\n", " \"\"\"Quick chat function using the wrapper\"\"\"\n", " try:\n", " messages = [{\"role\": \"user\", \"content\": prompt}]\n", " result = llm_api.chat(messages, max_tokens=max_tokens, temperature=temperature)\n", " return result['choices'][0]['message']['content']\n", " except Exception as e:\n", " return f\"Error: {e}\"\n", "\n", "def quick_complete(prompt: str, max_tokens: int = 50, temperature: float = 0.7) -> str:\n", " \"\"\"Quick completion function using the wrapper\"\"\"\n", " try:\n", " result = llm_api.complete(prompt, max_tokens=max_tokens, temperature=temperature)\n", " return result['choices'][0]['text']\n", " except Exception as e:\n", " return f\"Error: {e}\"\n", "\n", "def get_api_info() -> Dict[str, Any]:\n", " \"\"\"Get API information\"\"\"\n", " try:\n", " models = llm_api.list_models()\n", " return {\n", " \"endpoint\": llm_api.base_url,\n", " \"health\": llm_api.health_check(),\n", " \"model_count\": len(models),\n", " \"current_model\": llm_api.model_name,\n", " \"available_models\": [m.get('id', 'Unknown') for m in models[:5]] # First 5\n", " }\n", " except Exception as e:\n", " return {\"error\": str(e)}\n", "\n", "# Test the utility functions\n", "print(\"๐Ÿงช Testing utility functions:\")\n", "\n", "print(\"\\n1. Quick chat:\")\n", "chat_result = quick_chat(\"What is the meaning of life?\", max_tokens=50)\n", "print(f\"Response: {chat_result}\")\n", "\n", "print(\"\\n2. Quick completion:\")\n", "completion_result = quick_complete(\"The best programming language is\", max_tokens=30)\n", "print(f\"Response: {completion_result}\")\n", "\n", "print(\"\\n3. API info:\")\n", "api_info = get_api_info()\n", "for key, value in api_info.items():\n", " print(f\" {key}: {value}\")\n", "\n", "print(\"\\n=== Summary ===\")\n", "print(\"โœ… LlamaCPP API Wrapper Examples Complete!\")\n", "print(f\"๐Ÿ”— API Endpoint: {llm_api.base_url}\")\n", "print(f\"๐Ÿค– Model: {llm_api.model_name}\")\n", "print(\"\\n๐Ÿ”ง Wrapper Features Demonstrated:\")\n", "print(\" - โœ… Automatic model discovery\")\n", "print(\" - โœ… Health checking\")\n", "print(\" - โœ… Text completion\")\n", "print(\" - โœ… Chat completion\")\n", "print(\" - โœ… Streaming completion\")\n", "print(\" - โœ… Streaming chat\")\n", "print(\" - โœ… Error handling\")\n", "print(\" - โœ… Clean, consistent API\")\n", "\n", "print(\"\\n๐Ÿ’ก Wrapper Benefits:\")\n", "print(\" - Simple, intuitive interface\")\n", "print(\" - Automatic error handling\")\n", "print(\" - Type hints for better IDE support\")\n", "print(\" - Consistent parameter naming\")\n", "print(\" - Built-in streaming support\")\n", "print(\" - Connection management\")" ] }, { "cell_type": "markdown", "id": "451babc6", "metadata": {}, "source": [ "# LlamaCPP API Wrapper Examples\n", "\n", "This notebook demonstrates a **custom Python wrapper** for the LlamaCPP API running at `http://ollama.lan:8080/`.\n", "\n", "## ๐ŸŽฏ Wrapper Benefits\n", "\n", "### Why Use This Wrapper?\n", "- **๐Ÿ Pythonic Interface**: Clean, object-oriented API design\n", "- **๐Ÿ”„ Auto-Discovery**: Automatically finds available models\n", "- **๐Ÿ›ก๏ธ Error Handling**: Robust error management and retries\n", "- **๐Ÿ“ก Streaming Support**: Built-in streaming for real-time responses\n", "- **๐ŸŽ›๏ธ Consistent API**: Unified interface for all operations\n", "- **โšก Connection Management**: Efficient HTTP connection handling\n", "\n", "### vs. Raw HTTP Requests:\n", "- **No Manual JSON**: Automatic request/response handling\n", "- **Type Safety**: Type hints for better development experience\n", "- **Error Recovery**: Graceful handling of network issues\n", "- **Code Reusability**: Clean methods for common operations\n", "\n", "## ๐Ÿ”ง Wrapper Features\n", "\n", "### Core Methods:\n", "- `health_check()` - Check API availability\n", "- `list_models()` - Discover available models\n", "- `complete()` - Text completion\n", "- `chat()` - Chat completion\n", "- `stream_complete()` - Streaming text generation\n", "- `stream_chat()` - Streaming chat responses\n", "\n", "### Advanced Features:\n", "- **Automatic Model Discovery**: Finds and uses available models\n", "- **Flexible Parameters**: Support for temperature, top_p, stop sequences\n", "- **Streaming Iterators**: Python generators for real-time responses\n", "- **Connection Pooling**: Efficient HTTP connection reuse\n", "- **Timeout Management**: Configurable request timeouts\n", "\n", "## ๐Ÿ“Š Examples Included:\n", "\n", "1. **API Health & Model Discovery** - Check connection and find models\n", "2. **Simple Text Completion** - Basic text generation\n", "3. **Chat Completion** - Conversational AI with system prompts\n", "4. **Streaming Completion** - Real-time text generation\n", "5. **Streaming Chat** - Real-time conversation\n", "6. **Embeddings Test** - Vector representations (if supported)\n", "7. **Utility Functions** - Helper methods for common tasks\n", "\n", "## ๐Ÿš€ Getting Started:\n", "\n", "1. Ensure LlamaCPP server is running at `http://ollama.lan:8080/`\n", "2. Run the wrapper initialization cell\n", "3. Execute examples to see the wrapper in action\n", "\n", "The wrapper automatically discovers available models and provides a clean, consistent interface for all LlamaCPP API operations!" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" } }, "nbformat": 4, "nbformat_minor": 5 }