التحسين في التعلم العميق

import torch
import torch.nn as nn
import torch.optim as optim
from torch.cuda.amp import GradScaler, autocast
from torch.utils.data import TensorDataset, DataLoader

class="tok-comment"># ── Minimal model + dataloader for the demo ────────────────────────────
class MyModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(class="tok-num">16, class="tok-num">64), nn.ReLU(), nn.Linear(class="tok-num">64, class="tok-num">10))
    def forward(self, x): return self.net(x)

X_data = torch.randn(class="tok-num">512, class="tok-num">16)
y_data = torch.randint(class="tok-num">0, class="tok-num">10, (class="tok-num">512,))
train_loader = DataLoader(TensorDataset(X_data, y_data), batch_size=class="tok-num">32, shuffle=True)

def train_one_epoch(model, loader, optimizer, scaler, scheduler, device, clip_norm=class="tok-num">1.0):
    model.train()
    total_loss = class="tok-num">0.0
    for batch_idx, (X, y) in enumerate(loader):
        X, y = X.to(device), y.to(device)
        optimizer.zero_grad(set_to_none=True)   class="tok-comment"># faster than zero_grad()

        class="tok-comment"># ── Mixed precision forward pass ──────────────────────────────────────
        with autocast():
            logits = model(X)
            loss   = nn.functional.cross_entropy(logits, y, label_smoothing=class="tok-num">0.1)

        class="tok-comment"># ── Scaled backward + gradient clipping ──────────────────────────────
        scaler.scale(loss).backward()
        scaler.unscale_(optimizer)
        torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
        scaler.step(optimizer)
        scaler.update()
        scheduler.step()                         class="tok-comment"># step per batch for OneCycleLR

        total_loss += loss.item()

    return total_loss / len(loader)

class="tok-comment"># ── Setup ─────────────────────────────────────────────────────────────────────
device = torch.device(class="tok-str">"cuda" if torch.cuda.is_available() else class="tok-str">"cpu")
model  = MyModel().to(device)

class="tok-comment"># AdamW (Adam with decoupled weight decay — better than Adam+L2)
optimizer = optim.AdamW(model.parameters(), lr=class="tok-num">3e-4, weight_decay=class="tok-num">1e-2)

class="tok-comment"># OneCycle LR: warmup + cosine anneal in one schedule
scheduler = optim.lr_scheduler.OneCycleLR(
    optimizer,
    max_lr=class="tok-num">3e-4,
    total_steps=class="tok-num">100 * len(train_loader),    class="tok-comment"># epochs × steps_per_epoch
    pct_start=class="tok-num">0.05,                          class="tok-comment"># class="tok-num">5% warmup
    anneal_strategy=class="tok-str">"cos",
)

scaler = GradScaler()                        class="tok-comment"># for mixed precision

class="tok-comment"># ── Optimizer comparison ──────────────────────────────────────────────────────
import torch.optim as optim

class="tok-comment"># SGD + Momentum (strong for fine-tuning pretrained CNNs)
sgd = optim.SGD(model.parameters(), lr=class="tok-num">0.01, momentum=class="tok-num">0.9, weight_decay=class="tok-num">1e-4, nesterov=True)

class="tok-comment"># Adam (default for most deep learning tasks)
adam = optim.Adam(model.parameters(), lr=class="tok-num">3e-4, betas=(class="tok-num">0.9, class="tok-num">0.999))

class="tok-comment"># AdamW (Adam with properly decoupled weight decay — recommended by modern best practices)
adamw = optim.AdamW(model.parameters(), lr=class="tok-num">3e-4, weight_decay=class="tok-num">0.01)

class="tok-comment"># ── Batch Normalization example ───────────────────────────────────────────────
class ConvBlock(nn.Module):
    def __init__(self, in_ch, out_ch):
        super().__init__()
        self.block = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, class="tok-num">3, padding=class="tok-num">1, bias=False),
            nn.BatchNorm2d(out_ch),   class="tok-comment"># bias=False because BN has its own β
            nn.ReLU(inplace=True),
        )
    def forward(self, x): return self.block(x)

class="tok-comment"># ── Layer Normalization (Transformers prefer LayerNorm) ───────────────────────
class TransformerBlock(nn.Module):
    def __init__(self, d_model, n_heads):
        super().__init__()
        self.attn    = nn.MultiheadAttention(d_model, n_heads, batch_first=True)
        self.ff      = nn.Sequential(nn.Linear(d_model, d_model*class="tok-num">4), nn.GELU(), nn.Linear(d_model*class="tok-num">4, d_model))
        self.norm1   = nn.LayerNorm(d_model)    class="tok-comment"># pre-norm (more stable than post-norm)
        self.norm2   = nn.LayerNorm(d_model)
        self.drop    = nn.Dropout(class="tok-num">0.1)
    def forward(self, x):
        class="tok-comment"># Pre-norm architecture (used in GPT-class="tok-num">2+, better gradient flow)
        x = x + self.drop(self.attn(self.norm1(x), self.norm1(x), self.norm1(x))[class="tok-num">0])
        x = x + self.drop(self.ff(self.norm2(x)))
        return x

class="tok-comment"># ── Learning rate finder ──────────────────────────────────────────────────────
class="tok-comment"># pip install torch-lr-finder
from torch_lr_finder import LRFinder
finder = LRFinder(model, optimizer, nn.CrossEntropyLoss(), device=device)
finder.range_test(train_loader, end_lr=class="tok-num">10, num_iter=class="tok-num">100)
finder.plot()   class="tok-comment"># look for the steepest descent — that's your max_lr