M24: What's Next — The Open Source Frontier

import base64
from openai import OpenAI

# WHY: same client you've used throughout this track — no new dependencies
client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

# WHAT: read and encode the image as a base64 data URI
def encode_image(path: str) -> str:
    with open(path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

def analyze_image(image_path: str, question: str) -> str:
    b64 = encode_image(image_path)

    # WHAT: content is an array — one image_url item, one text item
    # GOTCHA: the image_url.url must be a valid data URI with the correct MIME type
    response = client.chat.completions.create(
        model="llava",   # or "llama4:scout" for better quality
        messages=[
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:image/jpeg;base64,{b64}"
                        }
                    },
                    {
                        "type": "text",
                        "text": question
                    }
                ]
            }
        ],
        max_tokens=512
    )
    return response.choices[0].message.content

# Example usage
if __name__ == "__main__":
    result = analyze_image("screenshot.png", "What UI elements are visible? List them.")
    print(result)

# pip install unsloth trl datasets
from unsloth import FastLanguageModel
from trl import SFTTrainer
from transformers import TrainingArguments
from datasets import load_dataset

# WHAT: load Mistral 7B in 4-bit (QLoRA) — fits in 6-8 GB VRAM
# WHY: load_in_4bit=True triggers the quantized base model
# GOTCHA: first run downloads ~4 GB; ensure HF_HOME has space
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/mistral-7b-bnb-4bit",
    max_seq_length=4096,
    load_in_4bit=True,
)

# WHAT: attach LoRA adapters to attention projection matrices
# WHY: q_proj and v_proj are the highest-impact targets for most tasks
# GOTCHA: larger r (rank) = more capacity but more VRAM; r=16 is a good default
model = FastLanguageModel.get_peft_model(
    model,
    r=16,               # rank of the adapter matrices
    lora_alpha=16,      # scaling factor — usually equal to r
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    lora_dropout=0.0,   # 0 is fine for most fine-tuning tasks
    bias="none",
)

# WHAT: load your training data (JSONL with "text" field containing full conversations)
dataset = load_dataset("json", data_files="my_domain_data.jsonl", split="train")

# WHAT: train with HuggingFace SFTTrainer (supervised fine-tuning)
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=4096,
    args=TrainingArguments(
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        warmup_steps=10,
        num_train_epochs=3,
        learning_rate=2e-4,
        logging_steps=25,
        output_dir="./lora-output",
        fp16=True,
    ),
)
trainer.train()

# WHAT: export to GGUF so Ollama can load it
# WHY: GGUF is the format ollama uses natively — no conversion step needed
# GOTCHA: q4_k_m gives the best size/quality tradeoff for most tasks
model.save_pretrained_gguf(
    "my-domain-model",
    tokenizer,
    quantization_method="q4_k_m"
)
print("Exported: my-domain-model-unsloth.Q4_K_M.gguf")

# Modelfile — place in the same directory as the .gguf file

# WHAT: points Ollama at the exported GGUF file
FROM ./my-domain-model-unsloth.Q4_K_M.gguf

# WHAT: system prompt baked into every conversation
# WHY: reduces per-request token cost; sets domain context permanently
SYSTEM """You are a specialized assistant for [Your Domain].
Use precise domain terminology. Always cite relevant standards when applicable.
Never use generic language when domain-specific phrasing exists."""

# WHAT: set inference parameters
PARAMETER temperature 0.3
PARAMETER top_p 0.9
PARAMETER num_ctx 4096

# Register with Ollama — runs once
ollama create my-domain-model -f Modelfile

# Test immediately
ollama run my-domain-model "What is a 510(k) clearance?"

# Use via API exactly like any other model
# client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")
# response = client.chat.completions.create(model="my-domain-model", ...)

import re
from openai import OpenAI

# Pull once before first run: ollama pull deepseek-r1:7b
client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

def think_and_answer(problem: str, show_thinking: bool = False) -> dict:
    """
    Call DeepSeek-R1.
    Returns {"thinking": str, "answer": str}.
    Set show_thinking=True to debug reasoning; False for end-user responses.
    """
    response = client.chat.completions.create(
        model="deepseek-r1:7b",
        messages=[
            {
                "role": "user",
                "content": problem
            }
        ],
        max_tokens=2048,
        temperature=0.6   # R1 works well at 0.5-0.7 for reasoning tasks
    )

    full_content = response.choices[0].message.content or ""

    # WHAT: extract the thinking trace
    # GOTCHA: some R1 variants use different tag formats — check with your version
    think_match = re.search(r"<think>(.*?)</think>", full_content, re.DOTALL)
    thinking = think_match.group(1).strip() if think_match else ""

    # WHAT: strip thinking tags to get the clean final answer
    answer = re.sub(r"<think>.*?</think>", "", full_content, flags=re.DOTALL).strip()

    return {
        "thinking": thinking if show_thinking else "[hidden]",
        "answer": answer
    }

# Example: a logic problem that benefits from chain-of-thought
result = think_and_answer(
    "A farmer has 17 sheep. All but 9 die. How many are left? "
    "Explain your reasoning step by step.",
    show_thinking=True  # set to False for production
)
print("THINKING:\n", result["thinking"])
print("\nANSWER:\n", result["answer"])