api: "YAML JSON TOON Database"
version: "1.0.0"
format: "json"
dataset:
  id: 61
  slug: "deep-learning-optimizers"
  title: "Deep Learning Optimizers"
  description: "Optimization algorithms for training neural networks: SGD, Momentum, NAG, Adam, AdamW, RMSprop, AdaGrad, and learning rate scheduling strategies."
  category: "AI & ML Terms"
  category_slug: "ai-ml-terms"
  tags: "deep-learning,optimizers,sgd,adam,adamw,rmsprop,learning-rate,momentum"
  view_count: 0
  created_at: 1778695229
  updated_at: 1778695229
data:
  optimizers:
    - name: "SGD (Stochastic Gradient Descent)"
      year: "1951 (Robbins-Monro)"
      update: "θ = θ - α * ∇L(θ)"
      pros:
        - "Simple, well-understood"
        - "Good generalization"
        - "No additional hyperparameters"
      cons:
        - "Slow convergence on ill-conditioned problems"
        - "Requires careful learning rate tuning"
        - "No per-parameter adaptation"
      best_for: "When generalization matters more than speed; convex or nearly convex problems"
      notes: "Still the gold standard for many vision tasks; often generalizes better than adaptive methods"
    - name: "SGD + Momentum"
      year: "1964 (Polyak)"
      update: "v = γv + α∇L(θ); θ = θ - v"
      pros:
        - "Accelerates convergence in relevant direction"
        - "Reduces oscillation"
        - "Damps oscillations across ravines"
      cons:
        - "Adds one hyperparameter (γ, typically 0.9)"
        - "May overshoot minima"
      best_for: "Most deep learning tasks; default choice for CNNs"
      notes: "Momentum term accumulates gradient in consistent directions; γ=0.9 is standard"
    - name: "NAG (Nesterov Accelerated Gradient)"
      year: "1983"
      update: "v = γv + α∇L(θ - γv); θ = θ - v"
      pros:
        - "Lookahead correction prevents overshooting"
        - "Better convergence rate than momentum"
        - "Theoretically optimal for convex functions"
      cons:
        - "Slightly more complex"
        - "Marginal practical improvement over momentum"
      best_for: "RNNs, problems with high curvature"
      notes: "Computes gradient at approximate future position rather than current position"
    - name: "AdaGrad (Adaptive Gradient)"
      year: "2011 (Duchi et al.)"
      update: "G = G + ∇L²; θ = θ - α/(√G + ε) * ∇L"
      pros:
        - "Per-parameter learning rates"
        - "No manual LR tuning per parameter"
        - "Excellent for sparse features"
      cons:
        - "Monotonically decreasing LR (can become too small)"
        - "Not suitable for non-convex deep learning"
      best_for: "Sparse features (NLP, recommender systems), convex optimization"
      notes: "Accumulates squared gradients; LR shrinks to zero eventually — problematic for deep nets"
    - name: "RMSprop (Root Mean Square Propagation)"
      year: "2012 (Hinton)"
      update: "E[g²] = γE[g²] + (1-γ)∇L²; θ = θ - α/(√E[g²] + ε) * ∇L"
      pros:
        - "Fixes AdaGrad's vanishing LR problem"
        - "Per-parameter adaptation"
        - "Works well for RNNs"
      cons:
        - "Adds γ hyperparameter (typically 0.9)"
        - "Can still get stuck in local minima"
      best_for: "RNNs, non-stationary objectives, online learning"
      notes: "Uses exponential moving average of squared gradients instead of cumulative sum"
    - name: "Adam (Adaptive Moment Estimation)"
      year: "2014 (Kingma & Ba)"
      update: "m = β₁m + (1-β₁)∇L; v = β₂v + (1-β₂)∇L²; θ = θ - α * m̂/(√v̂ + ε)"
      pros:
        - "Combines momentum + RMSprop"
        - "Bias correction for early steps"
        - "Minimal hyperparameter tuning needed"
        - "Works well out of the box"
      cons:
        - "May generalize worse than SGD"
        - "Memory overhead (2x gradients for m and v)"
        - "Can converge to sharp minima"
      best_for: "Default optimizer for most deep learning; fast prototyping; NLP, vision, RL"
      notes: "Default: β₁=0.9, β₂=0.999, ε=1e-8, α=0.001. Most widely used optimizer."
    - name: "AdamW (Adam with Decoupled Weight Decay)"
      year: "2017 (Loshchilov & Hutter)"
      update: "θ = θ - α * (m̂/(√v̂ + ε) + λθ)"
      pros:
        - "Proper weight decay implementation (decoupled from adaptive LR)"
        - "Better generalization than Adam"
        - "Standard for Transformer training"
      cons:
        - "One more hyperparameter (weight decay λ)"
        - "Still adaptive method generalization gap vs SGD"
      best_for: "Transformers (BERT, GPT), vision transformers, any architecture with weight decay"
      notes: "Weight decay applied outside adaptive LR computation; default for most modern architectures"
    - name: "LAMB / LARS"
      year: "2019"
      update: "Trust ratio × adaptive step (layer-wise adaptive scaling)"
      pros:
        - "Enables very large batch training (32K+)"
        - "Layer-wise LR adaptation"
        - "Stable at extreme batch sizes"
      cons:
        - "More complex"
        - "Less tested across architectures"
        - "Overkill for small batches"
      best_for: "Large-batch distributed training (TPU pods, multi-node GPU), BERT pretraining at scale"
      notes: "LARS (Layer-wise Adaptive Rate Scaling) and LAMB (Large Batch Adam) enable batch sizes that would diverge with standard Adam"