{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 501,
        "slug": "transfer-learning",
        "title": "Transfer Learning and Fine-Tuning",
        "description": "Transfer learning strategies: feature extraction, fine-tuning, domain adaptation, multi-task learning, prompt tuning, LoRA, and pretraining paradigms.",
        "category": "AI & ML Terms",
        "category_slug": "ai-ml-terms",
        "tags": "transfer-learning,fine-tuning,feature-extraction,domain-adaptation,lora,prompt-tuning,pretraining",
        "view_count": 4,
        "created_at": 1781275786,
        "updated_at": 1781275786
    },
    "data": {
        "strategies": [
            {
                "name": "Feature Extraction (Frozen Backbone)",
                "approach": "Use pretrained model as fixed feature extractor; train only new classification head",
                "when_to_use": [
                    "Small target dataset",
                    "Target domain similar to source",
                    "Limited compute resources"
                ],
                "pros": [
                    "Fast training",
                    "No catastrophic forgetting",
                    "Minimal overfitting risk"
                ],
                "cons": [
                    "Limited adaptation to target domain",
                    "Features may not be optimal for new task"
                ],
                "example": "ResNet-50 pretrained on ImageNet \u2192 freeze all layers \u2192 train only final FC layer on custom dataset"
            },
            {
                "name": "Fine-Tuning (Full Model)",
                "approach": "Unfreeze all layers; continue training entire model on target data with lower learning rate",
                "when_to_use": [
                    "Large target dataset",
                    "Target domain differs from source",
                    "Maximum performance needed"
                ],
                "pros": [
                    "Full adaptation to target domain",
                    "Best possible performance"
                ],
                "cons": [
                    "Risk of catastrophic forgetting",
                    "Requires more data",
                    "Computationally expensive"
                ],
                "example": "BERT pretrained \u2192 fine-tune all 110M parameters on SQuAD QA dataset with LR=2e-5"
            },
            {
                "name": "Layer-wise Learning Rate Decay",
                "approach": "Assign lower LR to earlier layers, higher LR to later layers during fine-tuning",
                "when_to_use": [
                    "Medium-sized target dataset",
                    "Want to preserve generic low-level features"
                ],
                "pros": [
                    "Balances preservation and adaptation",
                    "Reduces forgetting of early features"
                ],
                "cons": [
                    "Adds hyperparameter (decay rate)",
                    "Requires tuning"
                ],
                "example": "LR(layer_i) = base_lr \u00d7 decay^(depth - i); typical decay=0.9-0.95"
            },
            {
                "name": "Progressive Unfreezing",
                "approach": "Start with frozen backbone + trainable head; gradually unfreeze layers from top to bottom",
                "when_to_use": [
                    "Small target dataset",
                    "Want controlled adaptation",
                    "Avoid catastrophic forgetting"
                ],
                "pros": [
                    "Gradual adaptation reduces shock to pretrained weights",
                    "Better stability"
                ],
                "cons": [
                    "Slower training process",
                    "More complex training loop"
                ],
                "example": "Phase 1: train head only. Phase 2: unfreeze last block + head. Phase 3: unfreeze all."
            },
            {
                "name": "Domain Adaptation",
                "approach": "Align feature distributions between source and target domains (e.g., adversarial training, MMD loss)",
                "when_to_use": [
                    "Source and target domains differ significantly",
                    "Labeled target data scarce",
                    "Domain shift is known issue"
                ],
                "pros": [
                    "Explicitly addresses domain gap",
                    "Works with unlabeled target data"
                ],
                "cons": [
                    "More complex training",
                    "Requires domain-specific techniques"
                ],
                "example": "DANN (Domain-Adversarial Neural Network): gradient reversal layer makes features domain-invariant"
            },
            {
                "name": "LoRA (Low-Rank Adaptation)",
                "approach": "Inject trainable low-rank decomposition matrices into attention\/FFN layers; freeze pretrained weights",
                "when_to_use": [
                    "Fine-tuning large models (LLMs, ViT)",
                    "Memory-constrained environments",
                    "Multiple downstream tasks from same base model"
                ],
                "pros": [
                    "Dramatically fewer trainable parameters (0.1-1% of original)",
                    "No inference latency overhead (merged into weights)",
                    "Multiple LoRA adapters can share same base model"
                ],
                "cons": [
                    "Slight accuracy drop vs full fine-tuning",
                    "Rank hyperparameter to tune"
                ],
                "example": "LLaMA 7B: full fine-tuning = 7B params; LoRA (r=8) = 4.7M params (0.07%)",
                "notes": "Used for most LLM fine-tuning (Alpaca, Vicuna, etc.)"
            },
            {
                "name": "Prompt Tuning \/ Prefix Tuning",
                "approach": "Learn soft prompts (continuous vectors) prepended to input; freeze entire model",
                "when_to_use": [
                    "Very large models where even LoRA is expensive",
                    "Multiple tasks from single model",
                    "When prompt engineering is natural fit"
                ],
                "pros": [
                    "Minimal trainable parameters (thousands vs billions)",
                    "No model modification needed",
                    "Easy to switch between tasks"
                ],
                "cons": [
                    "Lower performance than full fine-tuning",
                    "Only works well for very large models (10B+)"
                ],
                "example": "T5-11B: learn 100-token soft prompt (128-dim each) = 1.28M params vs 11B full model"
            },
            {
                "name": "Multi-Task Learning",
                "approach": "Train single model on multiple related tasks simultaneously with shared backbone",
                "when_to_use": [
                    "Multiple related tasks available",
                    "Data-scarce individual tasks",
                    "Want shared representations"
                ],
                "pros": [
                    "Improved generalization via inductive bias",
                    "Single model for multiple tasks",
                    "Data augmentation across tasks"
                ],
                "cons": [
                    "Task interference possible",
                    "Complex loss weighting",
                    "Harder to debug"
                ],
                "example": "BERT pretraining: MLM + NSP simultaneously. MT-DNN: shared BERT + task-specific heads."
            }
        ]
    }
}