LiteLLM provides a single API for 100+ LLM providers (OpenAI, Anthropic, local, etc.).

Installation

pip install litellm
litellm --model ollama/qwen2.5:7b

Server: http://localhost:8000

Unified API

from litellm import completion

# Single interface for all providers
response = completion(
    model="gpt-4",
    messages=[{"role": "user", "content": "Hello"}]
)

# Local model
response = completion(
    model="ollama/qwen2.5:7b",
    messages=[{"role": "user", "content": "Hello"}]
)

Routing

from litellm import Router

router = Router(
    model_list=[
        {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4"}},
        {"model_name": "qwen", "litellm_params": {"model": "ollama/qwen2.5:7b"}},
    ]
)

response = router.completion(
    model="gpt-4",
    messages=[...],
    fallback_list=["qwen"]  # Fallback if GPT-4 fails
)

Docker

services:
  litellm:
    image: ghcr.io/berriai/litellm:main
    ports:
      - "8000:8000"
    environment:
      OPENAI_API_KEY: your-key

Sources