{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 61,
        "slug": "deep-learning-optimizers",
        "title": "Deep Learning Optimizers",
        "description": "Optimization algorithms for training neural networks: SGD, Momentum, NAG, Adam, AdamW, RMSprop, AdaGrad, and learning rate scheduling strategies.",
        "category": "AI & ML Terms",
        "category_slug": "ai-ml-terms",
        "tags": "deep-learning,optimizers,sgd,adam,adamw,rmsprop,learning-rate,momentum",
        "view_count": 0,
        "created_at": 1778695229,
        "updated_at": 1778695229
    },
    "data": {
        "optimizers": [
            {"name": "SGD (Stochastic Gradient Descent)", "year": "1951 (Robbins-Monro)", "update": "θ = θ - α * ∇L(θ)", "pros": ["Simple, well-understood", "Good generalization", "No additional hyperparameters"], "cons": ["Slow convergence on ill-conditioned problems", "Requires careful learning rate tuning", "No per-parameter adaptation"], "best_for": "When generalization matters more than speed; convex or nearly convex problems", "notes": "Still the gold standard for many vision tasks; often generalizes better than adaptive methods"},
            {"name": "SGD + Momentum", "year": "1964 (Polyak)", "update": "v = γv + α∇L(θ); θ = θ - v", "pros": ["Accelerates convergence in relevant direction", "Reduces oscillation", "Damps oscillations across ravines"], "cons": ["Adds one hyperparameter (γ, typically 0.9)", "May overshoot minima"], "best_for": "Most deep learning tasks; default choice for CNNs", "notes": "Momentum term accumulates gradient in consistent directions; γ=0.9 is standard"},
            {"name": "NAG (Nesterov Accelerated Gradient)", "year": "1983", "update": "v = γv + α∇L(θ - γv); θ = θ - v", "pros": ["Lookahead correction prevents overshooting", "Better convergence rate than momentum", "Theoretically optimal for convex functions"], "cons": ["Slightly more complex", "Marginal practical improvement over momentum"], "best_for": "RNNs, problems with high curvature", "notes": "Computes gradient at approximate future position rather than current position"},
            {"name": "AdaGrad (Adaptive Gradient)", "year": "2011 (Duchi et al.)", "update": "G = G + ∇L²; θ = θ - α/(√G + ε) * ∇L", "pros": ["Per-parameter learning rates", "No manual LR tuning per parameter", "Excellent for sparse features"], "cons": ["Monotonically decreasing LR (can become too small)", "Not suitable for non-convex deep learning"], "best_for": "Sparse features (NLP, recommender systems), convex optimization", "notes": "Accumulates squared gradients; LR shrinks to zero eventually — problematic for deep nets"},
            {"name": "RMSprop (Root Mean Square Propagation)", "year": "2012 (Hinton)", "update": "E[g²] = γE[g²] + (1-γ)∇L²; θ = θ - α/(√E[g²] + ε) * ∇L", "pros": ["Fixes AdaGrad's vanishing LR problem", "Per-parameter adaptation", "Works well for RNNs"], "cons": ["Adds γ hyperparameter (typically 0.9)", "Can still get stuck in local minima"], "best_for": "RNNs, non-stationary objectives, online learning", "notes": "Uses exponential moving average of squared gradients instead of cumulative sum"},
            {"name": "Adam (Adaptive Moment Estimation)", "year": "2014 (Kingma & Ba)", "update": "m = β₁m + (1-β₁)∇L; v = β₂v + (1-β₂)∇L²; θ = θ - α * m̂/(√v̂ + ε)", "pros": ["Combines momentum + RMSprop", "Bias correction for early steps", "Minimal hyperparameter tuning needed", "Works well out of the box"], "cons": ["May generalize worse than SGD", "Memory overhead (2x gradients for m and v)", "Can converge to sharp minima"], "best_for": "Default optimizer for most deep learning; fast prototyping; NLP, vision, RL", "notes": "Default: β₁=0.9, β₂=0.999, ε=1e-8, α=0.001. Most widely used optimizer."},
            {"name": "AdamW (Adam with Decoupled Weight Decay)", "year": "2017 (Loshchilov & Hutter)", "update": "θ = θ - α * (m̂/(√v̂ + ε) + λθ)", "pros": ["Proper weight decay implementation (decoupled from adaptive LR)", "Better generalization than Adam", "Standard for Transformer training"], "cons": ["One more hyperparameter (weight decay λ)", "Still adaptive method generalization gap vs SGD"], "best_for": "Transformers (BERT, GPT), vision transformers, any architecture with weight decay", "notes": "Weight decay applied outside adaptive LR computation; default for most modern architectures"},
            {"name": "LAMB / LARS", "year": "2019", "update": "Trust ratio × adaptive step (layer-wise adaptive scaling)", "pros": ["Enables very large batch training (32K+)", "Layer-wise LR adaptation", "Stable at extreme batch sizes"], "cons": ["More complex", "Less tested across architectures", "Overkill for small batches"], "best_for": "Large-batch distributed training (TPU pods, multi-node GPU), BERT pretraining at scale", "notes": "LARS (Layer-wise Adaptive Rate Scaling) and LAMB (Large Batch Adam) enable batch sizes that would diverge with standard Adam"}
        ]
    }
}