Fine-Tuning LLMs

Original Layer: W [4096 × 4096] = 16M parameters
LoRA Delta:    A [4096 × 8] + B [8 × 4096] = 64K parameters

250× fewer parameters!

from unsloth import FastLanguageModel
from trl import SFTTrainer
from transformers import TrainingArguments
from datasets import load_dataset

# 1. Load model
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/llama-2-7b-bnb-4bit",
    max_seq_length=2048,
    load_in_4bit=True,  # QLoRA
)

# 2. Attach LoRA
model = FastLanguageModel.get_peft_model(
    model,
    r=16,  # Rank
    lora_alpha=16,
    target_modules=["q_proj", "v_proj"],
    lora_dropout=0.05,
    bias="none",
    use_gradient_checkpointing=True,
)

# 3. Load dataset
dataset = load_dataset("json", data_files="training_data.json")

# 4. Train
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset["train"],
    args=TrainingArguments(
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        warmup_steps=100,
        num_train_epochs=1,
        learning_rate=2e-4,
        output_dir="lora_output",
        optim="adamw_8bit",
        max_steps=500,  # ~30 minutes
    ),
    packing=True,
    max_seq_length=2048,
    dataset_text_field="text",
)

trainer.train()
model.save_pretrained("lora-weights")

[
  {
    "instruction": "Explain machine learning",
    "output": "ML is a subset of AI..."
  },
  {
    "instruction": "What is a transformer?",
    "output": "A transformer is an architecture..."
  }
]

Learning Rate:
  Full Fine-Tuning: 5e-5 (all params trained, small)
  LoRA: 2e-4 (only deltas, bigger)
  QLoRA: 2e-4 to 5e-4 (even bigger okay)

Epochs:
  Large dataset (>1000): 1-3
  Small dataset (<500): 3-5

Batch Size:
  Full FT: 8-16
  LoRA: 16-32
  QLoRA: 32-64

Warmup:
  5-10% of total training steps

Weight Decay:
  Standard: 0.01
  More regularization: 0.05
  Less regularization: 0.001

LoRA Rank:
  Small changes: r=4-8
  Standard: r=16 (industry default)
  Large changes: r=32-64

model.safetensors (140 GB) + adapter_model.safetensors (50 MB)
Pros: Multiple LoRAs on same model, reversible
Cons: Higher RAM at inference

from peft import AutoPeftModelForCausalLM

merged = AutoPeftModelForCausalLM.from_pretrained(
    "lora_weights",
    device_map="cpu",
    torch_dtype=torch.float16,
).merge_and_unload()

merged.save_pretrained("merged-model")

# Full fine-tuning setup (requires high-end GPU)
import deepspeed

def full_fine_tune():
    model = AutoModelForCausalLM.from_pretrained(
        "meta-llama/Llama-2-7b",
        torch_dtype=torch.float16,
        device_map="auto",  # Multi-GPU
    )

    training_args = TrainingArguments(
        output_dir="./finetuned_model",
        num_train_epochs=3,
        per_device_train_batch_size=8,
        per_device_eval_batch_size=8,
        gradient_accumulation_steps=4,
        learning_rate=5e-5,  # Lower for full FT
        warmup_steps=500,
        weight_decay=0.01,
        save_strategy="epoch",
        eval_strategy="epoch",
        deepspeed="ds_config.json",  # Multi-GPU memory management
    )

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
    )

    trainer.train()
    return model

def evaluate_fine_tuned_model(model, test_dataset, base_model=None):
    """Comprehensive evaluation."""

    results = {
        "perplexity": 0,
        "bleu_score": 0,
        "human_eval": 0,
        "task_accuracy": 0,
    }

    # 1. Perplexity (lower is better)
    with torch.no_grad():
        losses = []
        for batch in test_dataset:
            logits = model(**batch).logits
            loss = torch.nn.functional.cross_entropy(
                logits.view(-1, logits.shape[-1]),
                batch["labels"].view(-1)
            )
            losses.append(loss.item())

    results["perplexity"] = torch.exp(torch.tensor(losses).mean()).item()

    # 2. BLEU (for generative tasks)
    from nltk.translate.bleu_score import corpus_bleu

    predictions = []
    references = []
    for batch in test_dataset:
        pred = model.generate(batch["input_ids"], max_length=100)
        predictions.extend(tokenizer.batch_decode(pred))
        references.extend(batch["expected_output"])

    results["bleu_score"] = corpus_bleu(
        [[ref.split()] for ref in references],
        [pred.split() for pred in predictions]
    )

    # 3. Task-Specific Accuracy (if classification)
    correct = 0
    for batch in test_dataset:
        logits = model(**batch).logits
        predictions = logits.argmax(dim=-1)
        correct += (predictions == batch["labels"]).sum().item()

    results["task_accuracy"] = correct / len(test_dataset)

    # 4. Compare vs Base Model
    if base_model:
        base_perplexity = compute_perplexity(base_model, test_dataset)
        improvement = (base_perplexity - results["perplexity"]) / base_perplexity * 100
        results["improvement_vs_base"] = f"{improvement:.1f}%"

    return results

# BAD: 10K low-quality examples
[
    {"instruction": "hi", "output": "hello"},
    {"instruction": "test", "output": "test response"},
    {"instruction": "x", "output": "y"},
    ...  # Lots of junk
]

# GOOD: 1K curated examples
[
    {
        "instruction": "Explain photosynthesis in simple terms",
        "output": "Photosynthesis is the process where plants convert sunlight, water, and CO2 into glucose and oxygen. This provides energy for the plant's growth..."
    },
    {
        "instruction": "What is the capital of France?",
        "output": "Paris is the capital of France. It's located in northern France and has been the political and cultural center..."
    },
    ...  # All high-quality examples
]

# Result: 1K good → Better model than 10K bad

def recommend_hyperparameters(dataset_size, domain_specificity):
    """
    dataset_size: number of training examples
    domain_specificity: 0.0 (general) to 1.0 (very specific)
    """

    if dataset_size < 100:
        return {
            "method": "LoRA",
            "rank": 4,
            "learning_rate": 1e-4,
            "epochs": 5,
            "warning": "Too small! Expected 500+ examples"
        }

    elif dataset_size < 500:
        return {
            "method": "QLoRA",
            "rank": 8,
            "learning_rate": 2e-4,
            "epochs": 3,
            "batch_size": 32
        }

    elif dataset_size < 5000:
        lr = 2e-4 if domain_specificity > 0.7 else 5e-4
        return {
            "method": "LoRA",
            "rank": 16,
            "learning_rate": lr,
            "epochs": 1-2,
            "batch_size": 16
        }

    else:
        return {
            "method": "Full Fine-Tuning",
            "learning_rate": 5e-5,
            "epochs": 2-3,
            "batch_size": 8,
            "warmup_ratio": 0.1
        }

# Usage
config = recommend_hyperparameters(dataset_size=1000, domain_specificity=0.8)
print(config)
# → {"method": "LoRA", "rank": 16, "learning_rate": 0.0002, "epochs": 2, ...}

# Current (old fine-tuned model)
model_blue = load_model("production/llama-7b-v1")

# New (newly fine-tuned model)
model_green = load_model("production/llama-7b-v2")

# A/B test: 10% traffic to green
if random.random() < 0.1:
    response = model_green.generate(prompt)
else:
    response = model_blue.generate(prompt)

# Monitor green performance for 1 week
# If good: Swap to 100% green, make green → blue
# If bad: Rollback to all blue

# Combine multiple fine-tuned models
models = [
    load_model("lora/medical-domain"),
    load_model("lora/legal-domain"),
    load_model("lora/general-qa"),
]

def ensemble_generate(prompt, task_type):
    # Use task-appropriate model
    model = models[task_type]  # 0=medical, 1=legal, 2=general
    return model.generate(prompt)

# WRONG: Fine-tune on medical texts only
# Result: Model becomes good at medical Q&A
#         BUT forgets how to do general tasks!

# FIX: Mix datasets
training_data = [
    {"task": "medical", "examples": 500},
    {"task": "general_qa", "examples": 300},
    {"task": "coding", "examples": 200},
]
# Prevents catastrophic forgetting

# BAD: Training examples too formulaic
[
    {"input": "What is X?", "output": "X is..."},
    {"input": "What is Y?", "output": "Y is..."},
    ...  # 1000 of these exact patterns
]

# GOOD: Varied phrasings
[
    {"input": "What is photosynthesis?", "output": "..."},
    {"input": "Explain how photosynthesis works", "output": "..."},
    {"input": "Tell me about photosynthesis", "output": "..."},
]
# Model learns concepts, not memorizes patterns

# WRONG
train_on_all_data()  # No validation!

# RIGHT
train_data, val_data = split_dataset(dataset, test_size=0.2)

trainer = Trainer(
    model=model,
    train_dataset=train_data,
    eval_dataset=val_data,  # Evaluate during training
)

# Check validation loss doesn't increase (overfitting indicator)