M21B: Cloud Deployment

# Create VM with T4 GPU, Ubuntu 22.04, 50 GB SSD
# --maintenance-policy=TERMINATE is required for GPU VMs (no live migration)
gcloud compute instances create ollama-server \
  --zone=us-central1-a \
  --machine-type=n1-standard-4 \
  --accelerator=type=nvidia-tesla-t4,count=1 \
  --image-family=ubuntu-2204-lts \
  --image-project=ubuntu-os-cloud \
  --boot-disk-size=50GB \
  --boot-disk-type=pd-ssd \
  --maintenance-policy=TERMINATE \
  --metadata=startup-script='#!/bin/bash
    # Install CUDA drivers (required for T4 GPU access)
    apt-get install -y linux-headers-$(uname -r)
    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb -o /tmp/cuda-keyring.deb
    dpkg -i /tmp/cuda-keyring.deb
    apt-get update -y && apt-get install -y cuda-drivers
    # Install Ollama
    curl -fsSL https://ollama.com/install.sh | sh
    # Pull Mistral (4-bit, fits in T4 16 GB VRAM)
    sudo -u ubuntu ollama pull mistral
  '

# Verify the instance is running
gcloud compute instances describe ollama-server --zone=us-central1-a \
  --format="value(status)"
# Expected: RUNNING

# SSH into the VM
gcloud compute ssh ollama-server --zone=us-central1-a

# Inside the VM: write a systemd unit file for Ollama
sudo tee /etc/systemd/system/ollama.service > /dev/null <<'EOF'
[Unit]
Description=Ollama LLM Server
After=network.target

[Service]
# Run as the ubuntu user so model cache goes to /home/ubuntu/.ollama
User=ubuntu
ExecStart=/usr/local/bin/ollama serve
# Only bind to localhost — the SSH tunnel handles external access
Environment="OLLAMA_HOST=127.0.0.1:11434"
Restart=always
RestartSec=5
# Give the GPU time to initialise at startup
TimeoutStartSec=60

[Install]
WantedBy=multi-user.target
EOF

# Enable and start the service
sudo systemctl daemon-reload
sudo systemctl enable ollama
sudo systemctl start ollama

# Verify it is running
sudo systemctl status ollama
# Expected: Active: active (running)

# Open SSH tunnel in background (-N = no remote command, -f = background)
# -L 11434:localhost:11434 = forward local :11434 → VM's localhost:11434
gcloud compute ssh ollama-server --zone=us-central1-a \
  -- -L 11434:localhost:11434 -N -f

# Now test it — your local port 11434 is forwarded to the cloud VM
curl http://localhost:11434/api/tags
# Expected: {"models": [{"name": "mistral:latest", ...}]}

# Step 1a: Create a security group that allows SSH only (no inbound 11434)
aws ec2 create-security-group \
  --group-name ollama-sg \
  --description "SSH-only access for Ollama VM"

# Allow SSH from your current IP only (replace with your IP or use 0.0.0.0/0 for any)
MY_IP=$(curl -s https://checkip.amazonaws.com)
aws ec2 authorize-security-group-ingress \
  --group-name ollama-sg \
  --protocol tcp --port 22 --cidr "${MY_IP}/32"

# Step 1b: Launch g4dn.xlarge with Deep Learning AMI (Ubuntu 20.04)
# ami-0c2d0b2d77ffd5a16 is the DL AMI in us-east-1 — check AWS Marketplace for your region
INSTANCE_ID=$(aws ec2 run-instances \
  --image-id ami-0c2d0b2d77ffd5a16 \
  --instance-type g4dn.xlarge \
  --key-name my-key \
  --security-groups ollama-sg \
  --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":50,"VolumeType":"gp3"}}]' \
  --query 'Instances[0].InstanceId' --output text)

echo "Instance ID: $INSTANCE_ID"

# Wait until running
aws ec2 wait instance-running --instance-ids "$INSTANCE_ID"

# Get the public IP
VM_IP=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" \
  --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)

echo "VM IP: $VM_IP"

# SSH and install Ollama (CUDA is already on the DL AMI)
ssh -i my-key.pem ubuntu@${VM_IP} \
  "curl -fsSL https://ollama.com/install.sh | sh && ollama pull mistral"

# Set up systemd service (same unit file as GCP lab)
ssh -i my-key.pem ubuntu@${VM_IP} 'sudo tee /etc/systemd/system/ollama.service > /dev/null' <<'EOF'
[Unit]
Description=Ollama LLM Server
After=network.target

[Service]
User=ubuntu
ExecStart=/usr/local/bin/ollama serve
Environment="OLLAMA_HOST=127.0.0.1:11434"
Restart=always
RestartSec=5
TimeoutStartSec=60

[Install]
WantedBy=multi-user.target
EOF

ssh -i my-key.pem ubuntu@${VM_IP} \
  "sudo systemctl daemon-reload && sudo systemctl enable ollama && sudo systemctl start ollama"

# Open SSH tunnel — same pattern as GCP, different SSH invocation
ssh -i my-key.pem -L 11434:localhost:11434 -N -f ubuntu@${VM_IP}

# Verify
curl http://localhost:11434/api/tags
# Expected: {"models":[{"name":"mistral:latest",...}]}

# cost_estimator.py
# WHAT: Estimate monthly inference cost across cloud providers
# WHY:  "Just use Groq" sounds right until you model your actual token volume
# GOTCHA: VM costs assume the VM runs continuously; use hours_per_day to model
#         cost for dev environments that only run during business hours

from dataclasses import dataclass, field
from typing import Optional

@dataclass
class ProviderCost:
    name: str
    # For managed APIs: cost per 1 million input tokens
    input_cost_per_mtok: Optional[float] = None
    # For managed APIs: cost per 1 million output tokens
    output_cost_per_mtok: Optional[float] = None
    # For self-hosted VMs: cost per hour (compute only, no per-token fee)
    hourly_rate: Optional[float] = None
    notes: str = ""

PROVIDER_COSTS = [
    ProviderCost("Ollama/Localhost", hourly_rate=0.0,   notes="Existing hardware, electricity not counted"),
    ProviderCost("GCP T4 VM",        hourly_rate=0.35,  notes="n1-standard-4 + T4; no per-token cost"),
    ProviderCost("AWS g4dn.xlarge",  hourly_rate=0.53,  notes="On-demand; no per-token cost"),
    ProviderCost("Vast.ai 3090",     hourly_rate=0.20,  notes="Spot price; availability varies"),
    ProviderCost("Lambda Labs A10",  hourly_rate=0.60,  notes="Reserved; guaranteed availability"),
    ProviderCost("Groq (free)",      input_cost_per_mtok=0.0,   output_cost_per_mtok=0.0,  notes="14,400 req/day limit"),
    ProviderCost("Groq (paid)",      input_cost_per_mtok=0.05,  output_cost_per_mtok=0.08, notes="Llama 3.3 70B pricing"),
    ProviderCost("Together AI",      input_cost_per_mtok=0.10,  output_cost_per_mtok=0.10, notes="Mistral 7B pricing"),
    ProviderCost("Fireworks AI",     input_cost_per_mtok=0.10,  output_cost_per_mtok=0.10, notes="Mistral 7B pricing"),
]

@dataclass
class EstimateResult:
    provider: str
    monthly_cost_usd: float
    notes: str
    recommended: bool = False

def estimate_monthly_costs(
    tokens_per_day: int,
    output_ratio: float = 0.3,   # fraction of tokens that are output
    hours_per_day: float = 24.0, # for VM-based providers: hours VM runs per day
) -> list[EstimateResult]:
    """
    Estimate monthly costs across all providers.

    Args:
        tokens_per_day:  Total tokens processed per day (input + output combined).
        output_ratio:    Fraction of tokens_per_day that are output tokens (0.0–1.0).
                         Defaults to 0.3 (typical for RAG / summarisation workloads).
        hours_per_day:   How many hours per day VM-based providers are running.
                         Use 8.0 for a dev machine, 24.0 for production.

    Returns:
        List of EstimateResult sorted by monthly cost.
    """
    input_tokens_per_day  = tokens_per_day * (1 - output_ratio)
    output_tokens_per_day = tokens_per_day * output_ratio
    days_per_month = 30.4  # average

    results: list[EstimateResult] = []

    for p in PROVIDER_COSTS:
        if p.hourly_rate is not None:
            # VM cost: hourly rate × hours/day × days/month
            monthly = p.hourly_rate * hours_per_day * days_per_month
        else:
            # Managed API: per-token billing
            input_mtok_per_month  = (input_tokens_per_day  * days_per_month) / 1_000_000
            output_mtok_per_month = (output_tokens_per_day * days_per_month) / 1_000_000
            monthly = (
                (p.input_cost_per_mtok  or 0) * input_mtok_per_month +
                (p.output_cost_per_mtok or 0) * output_mtok_per_month
            )

        results.append(EstimateResult(
            provider=p.name,
            monthly_cost_usd=round(monthly, 2),
            notes=p.notes,
        ))

    results.sort(key=lambda r: r.monthly_cost_usd)

    # Mark the lowest non-zero cost option as recommended
    for r in results:
        if r.monthly_cost_usd > 0:
            r.recommended = True
            break

    return results


def print_estimate(tokens_per_day: int, hours_per_day: float = 24.0) -> None:
    """Print a formatted cost comparison table."""
    results = estimate_monthly_costs(tokens_per_day, hours_per_day=hours_per_day)
    total_mtok_month = tokens_per_day * 30.4 / 1_000_000

    print(f"\nMonthly cost estimate for {tokens_per_day:,} tokens/day")
    print(f"({total_mtok_month:.1f}M tokens/month, VM uptime: {hours_per_day:.0f} hr/day)")
    print(f"{'Provider':<22} {'$/month':>9}  {'Notes'}")
    print("-" * 70)
    for r in results:
        tag = " ← recommended" if r.recommended else ""
        print(f"  {r.provider:<20} ${r.monthly_cost_usd:>8.2f}  {r.notes}{tag}")


if __name__ == "__main__":
    # Dev workload: 100K tokens/day, VM only 8 hours/day
    print_estimate(tokens_per_day=100_000, hours_per_day=8)

    # Production workload: 1M tokens/day, VM runs 24/7
    print_estimate(tokens_per_day=1_000_000, hours_per_day=24)