{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 396,
        "slug": "deep-learning-optimizers",
        "title": "Deep Learning Optimizers",
        "description": "Optimization algorithms for training neural networks: SGD, Momentum, NAG, Adam, AdamW, RMSprop, AdaGrad, and learning rate scheduling strategies.",
        "category": "AI & ML Terms",
        "category_slug": "ai-ml-terms",
        "tags": "deep-learning,optimizers,sgd,adam,adamw,rmsprop,learning-rate,momentum",
        "view_count": 3,
        "created_at": 1781275786,
        "updated_at": 1781275786
    },
    "data": {
        "optimizers": [
            {
                "name": "SGD (Stochastic Gradient Descent)",
                "year": "1951 (Robbins-Monro)",
                "update": "\u03b8 = \u03b8 - \u03b1 * \u2207L(\u03b8)",
                "pros": [
                    "Simple, well-understood",
                    "Good generalization",
                    "No additional hyperparameters"
                ],
                "cons": [
                    "Slow convergence on ill-conditioned problems",
                    "Requires careful learning rate tuning",
                    "No per-parameter adaptation"
                ],
                "best_for": "When generalization matters more than speed; convex or nearly convex problems",
                "notes": "Still the gold standard for many vision tasks; often generalizes better than adaptive methods"
            },
            {
                "name": "SGD + Momentum",
                "year": "1964 (Polyak)",
                "update": "v = \u03b3v + \u03b1\u2207L(\u03b8); \u03b8 = \u03b8 - v",
                "pros": [
                    "Accelerates convergence in relevant direction",
                    "Reduces oscillation",
                    "Damps oscillations across ravines"
                ],
                "cons": [
                    "Adds one hyperparameter (\u03b3, typically 0.9)",
                    "May overshoot minima"
                ],
                "best_for": "Most deep learning tasks; default choice for CNNs",
                "notes": "Momentum term accumulates gradient in consistent directions; \u03b3=0.9 is standard"
            },
            {
                "name": "NAG (Nesterov Accelerated Gradient)",
                "year": "1983",
                "update": "v = \u03b3v + \u03b1\u2207L(\u03b8 - \u03b3v); \u03b8 = \u03b8 - v",
                "pros": [
                    "Lookahead correction prevents overshooting",
                    "Better convergence rate than momentum",
                    "Theoretically optimal for convex functions"
                ],
                "cons": [
                    "Slightly more complex",
                    "Marginal practical improvement over momentum"
                ],
                "best_for": "RNNs, problems with high curvature",
                "notes": "Computes gradient at approximate future position rather than current position"
            },
            {
                "name": "AdaGrad (Adaptive Gradient)",
                "year": "2011 (Duchi et al.)",
                "update": "G = G + \u2207L\u00b2; \u03b8 = \u03b8 - \u03b1\/(\u221aG + \u03b5) * \u2207L",
                "pros": [
                    "Per-parameter learning rates",
                    "No manual LR tuning per parameter",
                    "Excellent for sparse features"
                ],
                "cons": [
                    "Monotonically decreasing LR (can become too small)",
                    "Not suitable for non-convex deep learning"
                ],
                "best_for": "Sparse features (NLP, recommender systems), convex optimization",
                "notes": "Accumulates squared gradients; LR shrinks to zero eventually \u2014 problematic for deep nets"
            },
            {
                "name": "RMSprop (Root Mean Square Propagation)",
                "year": "2012 (Hinton)",
                "update": "E[g\u00b2] = \u03b3E[g\u00b2] + (1-\u03b3)\u2207L\u00b2; \u03b8 = \u03b8 - \u03b1\/(\u221aE[g\u00b2] + \u03b5) * \u2207L",
                "pros": [
                    "Fixes AdaGrad's vanishing LR problem",
                    "Per-parameter adaptation",
                    "Works well for RNNs"
                ],
                "cons": [
                    "Adds \u03b3 hyperparameter (typically 0.9)",
                    "Can still get stuck in local minima"
                ],
                "best_for": "RNNs, non-stationary objectives, online learning",
                "notes": "Uses exponential moving average of squared gradients instead of cumulative sum"
            },
            {
                "name": "Adam (Adaptive Moment Estimation)",
                "year": "2014 (Kingma & Ba)",
                "update": "m = \u03b2\u2081m + (1-\u03b2\u2081)\u2207L; v = \u03b2\u2082v + (1-\u03b2\u2082)\u2207L\u00b2; \u03b8 = \u03b8 - \u03b1 * m\u0302\/(\u221av\u0302 + \u03b5)",
                "pros": [
                    "Combines momentum + RMSprop",
                    "Bias correction for early steps",
                    "Minimal hyperparameter tuning needed",
                    "Works well out of the box"
                ],
                "cons": [
                    "May generalize worse than SGD",
                    "Memory overhead (2x gradients for m and v)",
                    "Can converge to sharp minima"
                ],
                "best_for": "Default optimizer for most deep learning; fast prototyping; NLP, vision, RL",
                "notes": "Default: \u03b2\u2081=0.9, \u03b2\u2082=0.999, \u03b5=1e-8, \u03b1=0.001. Most widely used optimizer."
            },
            {
                "name": "AdamW (Adam with Decoupled Weight Decay)",
                "year": "2017 (Loshchilov & Hutter)",
                "update": "\u03b8 = \u03b8 - \u03b1 * (m\u0302\/(\u221av\u0302 + \u03b5) + \u03bb\u03b8)",
                "pros": [
                    "Proper weight decay implementation (decoupled from adaptive LR)",
                    "Better generalization than Adam",
                    "Standard for Transformer training"
                ],
                "cons": [
                    "One more hyperparameter (weight decay \u03bb)",
                    "Still adaptive method generalization gap vs SGD"
                ],
                "best_for": "Transformers (BERT, GPT), vision transformers, any architecture with weight decay",
                "notes": "Weight decay applied outside adaptive LR computation; default for most modern architectures"
            },
            {
                "name": "LAMB \/ LARS",
                "year": "2019",
                "update": "Trust ratio \u00d7 adaptive step (layer-wise adaptive scaling)",
                "pros": [
                    "Enables very large batch training (32K+)",
                    "Layer-wise LR adaptation",
                    "Stable at extreme batch sizes"
                ],
                "cons": [
                    "More complex",
                    "Less tested across architectures",
                    "Overkill for small batches"
                ],
                "best_for": "Large-batch distributed training (TPU pods, multi-node GPU), BERT pretraining at scale",
                "notes": "LARS (Layer-wise Adaptive Rate Scaling) and LAMB (Large Batch Adam) enable batch sizes that would diverge with standard Adam"
            }
        ]
    }
}