api: "YAML JSON TOON Database"
version: "1.0.0"
format: "json"
dataset:
  id: 64
  slug: "transfer-learning"
  title: "Transfer Learning and Fine-Tuning"
  description: "Transfer learning strategies: feature extraction, fine-tuning, domain adaptation, multi-task learning, prompt tuning, LoRA, and pretraining paradigms."
  category: "AI & ML Terms"
  category_slug: "ai-ml-terms"
  tags: "transfer-learning,fine-tuning,feature-extraction,domain-adaptation,lora,prompt-tuning,pretraining"
  view_count: 0
  created_at: 1778695229
  updated_at: 1778695229
data:
  strategies:
    - name: "Feature Extraction (Frozen Backbone)"
      approach: "Use pretrained model as fixed feature extractor; train only new classification head"
      when_to_use:
        - "Small target dataset"
        - "Target domain similar to source"
        - "Limited compute resources"
      pros:
        - "Fast training"
        - "No catastrophic forgetting"
        - "Minimal overfitting risk"
      cons:
        - "Limited adaptation to target domain"
        - "Features may not be optimal for new task"
      example: "ResNet-50 pretrained on ImageNet → freeze all layers → train only final FC layer on custom dataset"
    - name: "Fine-Tuning (Full Model)"
      approach: "Unfreeze all layers; continue training entire model on target data with lower learning rate"
      when_to_use:
        - "Large target dataset"
        - "Target domain differs from source"
        - "Maximum performance needed"
      pros:
        - "Full adaptation to target domain"
        - "Best possible performance"
      cons:
        - "Risk of catastrophic forgetting"
        - "Requires more data"
        - "Computationally expensive"
      example: "BERT pretrained → fine-tune all 110M parameters on SQuAD QA dataset with LR=2e-5"
    - name: "Layer-wise Learning Rate Decay"
      approach: "Assign lower LR to earlier layers, higher LR to later layers during fine-tuning"
      when_to_use:
        - "Medium-sized target dataset"
        - "Want to preserve generic low-level features"
      pros:
        - "Balances preservation and adaptation"
        - "Reduces forgetting of early features"
      cons:
        - "Adds hyperparameter (decay rate)"
        - "Requires tuning"
      example: "LR(layer_i) = base_lr × decay^(depth - i); typical decay=0.9-0.95"
    - name: "Progressive Unfreezing"
      approach: "Start with frozen backbone + trainable head; gradually unfreeze layers from top to bottom"
      when_to_use:
        - "Small target dataset"
        - "Want controlled adaptation"
        - "Avoid catastrophic forgetting"
      pros:
        - "Gradual adaptation reduces shock to pretrained weights"
        - "Better stability"
      cons:
        - "Slower training process"
        - "More complex training loop"
      example: "Phase 1: train head only. Phase 2: unfreeze last block + head. Phase 3: unfreeze all."
    - name: "Domain Adaptation"
      approach: "Align feature distributions between source and target domains (e.g., adversarial training, MMD loss)"
      when_to_use:
        - "Source and target domains differ significantly"
        - "Labeled target data scarce"
        - "Domain shift is known issue"
      pros:
        - "Explicitly addresses domain gap"
        - "Works with unlabeled target data"
      cons:
        - "More complex training"
        - "Requires domain-specific techniques"
      example: "DANN (Domain-Adversarial Neural Network): gradient reversal layer makes features domain-invariant"
    - name: "LoRA (Low-Rank Adaptation)"
      approach: "Inject trainable low-rank decomposition matrices into attention/FFN layers; freeze pretrained weights"
      when_to_use:
        - "Fine-tuning large models (LLMs, ViT)"
        - "Memory-constrained environments"
        - "Multiple downstream tasks from same base model"
      pros:
        - "Dramatically fewer trainable parameters (0.1-1% of original)"
        - "No inference latency overhead (merged into weights)"
        - "Multiple LoRA adapters can share same base model"
      cons:
        - "Slight accuracy drop vs full fine-tuning"
        - "Rank hyperparameter to tune"
      example: "LLaMA 7B: full fine-tuning = 7B params; LoRA (r=8) = 4.7M params (0.07%)"
      notes: "Used for most LLM fine-tuning (Alpaca, Vicuna, etc.)"
    - name: "Prompt Tuning / Prefix Tuning"
      approach: "Learn soft prompts (continuous vectors) prepended to input; freeze entire model"
      when_to_use:
        - "Very large models where even LoRA is expensive"
        - "Multiple tasks from single model"
        - "When prompt engineering is natural fit"
      pros:
        - "Minimal trainable parameters (thousands vs billions)"
        - "No model modification needed"
        - "Easy to switch between tasks"
      cons:
        - "Lower performance than full fine-tuning"
        - "Only works well for very large models (10B+)"
      example: "T5-11B: learn 100-token soft prompt (128-dim each) = 1.28M params vs 11B full model"
    - name: "Multi-Task Learning"
      approach: "Train single model on multiple related tasks simultaneously with shared backbone"
      when_to_use:
        - "Multiple related tasks available"
        - "Data-scarce individual tasks"
        - "Want shared representations"
      pros:
        - "Improved generalization via inductive bias"
        - "Single model for multiple tasks"
        - "Data augmentation across tasks"
      cons:
        - "Task interference possible"
        - "Complex loss weighting"
        - "Harder to debug"
      example: "BERT pretraining: MLM + NSP simultaneously. MT-DNN: shared BERT + task-specific heads."