为什么个人 AI 项目也该尽早加一层 LLM Gateway

# bad_app.py —— 业务里直接握多家 Key
import os
from openai import OpenAI
import anthropic

def chat(provider: str, prompt: str) -> str:
    if provider == "openai":
        c = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
        r = c.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": prompt}],
            timeout=60,
        )
        return r.choices[0].message.content or ""

    if provider == "anthropic":
        c = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
        r = c.messages.create(
            model="claude-sonnet-4-0",
            max_tokens=1024,
            messages=[{"role": "user", "content": prompt}],
        )
        return r.content[0].text

    if provider == "ollama":
        c = OpenAI(
            api_key="ollama",
            base_url="http://127.0.0.1:11434/v1",
        )
        r = c.chat.completions.create(
            model="qwen2.5:14b",
            messages=[{"role": "user", "content": prompt}],
        )
        return r.choices[0].message.content or ""

    raise ValueError(provider)

# good_client.py —— 业务只打网关
from openai import OpenAI

client = OpenAI(
    api_key="sk-local-dev",           # 网关发的虚拟 Key
    base_url="http://127.0.0.1:4000/v1",
)

resp = client.chat.completions.create(
    model="auto",  # 网关侧映射到真实模型
    messages=[
        {"role": "system", "content": "你是简洁的中文技术助手。"},
        {"role": "user", "content": "用三句话解释什么是 LLM Gateway。"},
    ],
    temperature=0.3,
)
print(resp.choices[0].message.content)
print("usage:", resp.usage)

model_list:
  # 本地优先：开发机 / 内网 GPU
  - model_name: local-qwen
    litellm_params:
      model: ollama/qwen2.5:14b
      api_base: http://host.docker.internal:11434
      rpm: 30

  # 云端兜底：本地挂了再走
  - model_name: cloud-mini
    litellm_params:
      model: openai/gpt-4o-mini
      api_key: os.environ/OPENAI_API_KEY

  # 对外暴露的「逻辑模型名」
  - model_name: auto
    litellm_params:
      model: ollama/qwen2.5:14b
      api_base: http://host.docker.internal:11434

router_settings:
  # 主模型失败时按顺序切换
  fallbacks:
    - auto: ["cloud-mini"]
    - local-qwen: ["cloud-mini"]

general_settings:
  master_key: sk-change-me-in-prod
  # 生产务必换掉，并用环境变量注入

services:
  litellm:
    image: ghcr.io/berriai/litellm:main-latest
    ports:
      - "127.0.0.1:4000:4000"   # 只绑本机，外面走 Nginx
    volumes:
      - ./config.yaml:/app/config.yaml:ro
    environment:
      OPENAI_API_KEY: ${OPENAI_API_KEY}
      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
    command: ["--config", "/app/config.yaml", "--port", "4000"]
    restart: unless-stopped

export OPENAI_API_KEY=sk-...
docker compose up -d

curl -s http://127.0.0.1:4000/v1/chat/completions \
  -H "Authorization: Bearer sk-change-me-in-prod" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "auto",
    "messages": [{"role":"user","content":"ping"}]
  }' | jq '.choices[0].message.content, .model, .usage'

import uuid
from openai import OpenAI

client = OpenAI(api_key="sk-local-dev", base_url="http://127.0.0.1:4000/v1")
rid = str(uuid.uuid4())

resp = client.chat.completions.create(
    model="auto",
    messages=[{"role": "user", "content": "总结这段报错：Connection reset by peer"}],
    extra_headers={"X-Request-Id": rid},
)
print(rid, resp.model, resp.usage)