Claude Code with Open Source & Local Models

{
  "env": {
    "ANTHROPIC_BASE_URL": "http://localhost:4000",
    "ANTHROPIC_API_KEY": "local-placeholder"
  }
}

# WHAT: Download a model and start its server
# WHY: The pull step caches weights locally (~4GB for a 7B model)
# GOTCHA: First pull can take 5-20 minutes depending on your connection

# Start the Ollama daemon (if not already running as a background service)
ollama serve &

# Pull a model (choose based on your available RAM)
ollama pull llama3.2          # 3.2B params — needs ~2GB RAM
ollama pull llama3.2:latest   # same thing — :latest is implicit
ollama pull qwen2.5-coder:7b  # 7B params — strong coder, needs ~5GB RAM
ollama pull qwen2.5-coder:32b # 32B params — best quality, needs ~20GB RAM

# Verify the model is available
ollama list

# Quick smoke test — interactive chat
ollama run llama3.2 "Write a Python function to reverse a string"

# Install LiteLLM proxy
pip install 'litellm[proxy]'

# Start the proxy pointed at your Ollama instance
litellm --model ollama/qwen2.5-coder:7b --port 4000

# In another terminal — launch Claude Code through the proxy
export ANTHROPIC_BASE_URL="http://localhost:4000"
export ANTHROPIC_API_KEY="local"
claude

# WHAT: LiteLLM config defining a routing table of models
# WHY: Lets you alias multiple backends under names Claude Code can request
# GOTCHA: model_name values here are what you pass to --model in Claude Code

model_list:

  # Fast local model for boilerplate and quick edits
  - model_name: local-fast
    litellm_params:
      model: ollama/llama3.2
      api_base: http://localhost:11434

  # Better quality local model for code-heavy tasks
  - model_name: local-coder
    litellm_params:
      model: ollama/qwen2.5-coder:7b
      api_base: http://localhost:11434

  # Cloud fallback when local isn't good enough
  - model_name: cloud-open
    litellm_params:
      model: openrouter/meta-llama/llama-3.3-70b-instruct
      api_key: "os.environ/OPENROUTER_API_KEY"

  # High-speed inference for interactive use
  - model_name: groq-fast
    litellm_params:
      model: groq/llama-3.3-70b-versatile
      api_key: "os.environ/GROQ_API_KEY"

litellm_settings:
  # Drop unsupported params (like temperature when using Ollama with some models)
  drop_params: true
  # Return full error messages for easier debugging
  set_verbose: false

general_settings:
  master_key: "local-master-key"  # Set any non-empty value for local use

# Start proxy using config file
litellm --config litellm-config.yaml --port 4000

# Now Claude Code can request any model by its model_name alias
export ANTHROPIC_BASE_URL="http://localhost:4000"
export ANTHROPIC_API_KEY="local-master-key"

# Use the fast local model for a quick task
claude --model local-fast "Add type annotations to all functions in utils.py"

# Switch to the better coder model for a harder task
claude --model local-coder "Refactor this class to use async/await throughout"

# Fall back to cloud when you need more horsepower
claude --model cloud-open "Design the architecture for a distributed rate limiter"

# WHAT: Launch a production-grade local inference server
# WHY: vLLM handles concurrent requests efficiently using PagedAttention
# GOTCHA: Requires an NVIDIA GPU with CUDA; minimum ~16GB VRAM for 7B models

pip install vllm

# Start the server (downloads model from HuggingFace on first run)
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-Coder-7B-Instruct \
    --port 8000 \
    --host 0.0.0.0

# Route Claude Code through LiteLLM → vLLM
litellm --model openai/Qwen2.5-Coder-7B-Instruct \
        --api_base http://localhost:8000/v1 \
        --api_key vllm \
        --port 4000

# Connect Claude Code
export ANTHROPIC_BASE_URL="http://localhost:4000"
export ANTHROPIC_API_KEY="local"
claude

# 1. Is Ollama running?
ollama list         # should list downloaded models
curl http://localhost:11434/api/tags   # should return JSON

# 2. Is LiteLLM proxy running?
curl http://localhost:4000/health      # should return {"status":"healthy"}

# 3. Can LiteLLM reach Ollama?
curl http://localhost:4000/v1/models   # should list available models

# 4. Is the Anthropic API format being accepted?
curl -X POST http://localhost:4000/v1/messages \
  -H "Content-Type: application/json" \
  -H "x-api-key: local-key" \
  -d '{
    "model": "ollama/llama3.2",
    "max_tokens": 64,
    "messages": [{"role": "user", "content": "Say hello"}]
  }'
# Should return a response in Anthropic format