Apprentissage par Renforcement

import numpy as np
import gymnasium as gym
from collections import deque
import torch, torch.nn as nn, torch.optim as optim

class="tok-comment"># ── class="tok-num">1. Q-Learning Tabulaire (FrozenLake) ─────────────────────────────────────
env = gym.make(class="tok-str">"FrozenLake-v1", is_slippery=False)
n_etats, n_actions = env.observation_space.n, env.action_space.n

Q       = np.zeros((n_etats, n_actions))
alpha   = class="tok-num">0.8          class="tok-comment"># taux dclass="tok-str">'apprentissage
gamma   = class="tok-num">0.95         class="tok-comment"># escompte
epsilon = class="tok-num">1.0          class="tok-comment"># taux d'exploration

for episode in range(class="tok-num">2000):
    s, _ = env.reset()
    termine = False
    while not termine:
        class="tok-comment"># ε-greedy
        if np.random.random() < epsilon:
            a = env.action_space.sample()
        else:
            a = np.argmax(Q[s])

        s_suiv, r, fin, tronque, _ = env.step(a)
        termine = fin or tronque

        class="tok-comment"># Mise à jour Q-learning (cible Bellman)
        cible_td = r + gamma * np.max(Q[s_suiv]) * (class="tok-num">1 - fin)
        Q[s, a] += alpha * (cible_td - Q[s, a])
        s = s_suiv

    epsilon = max(class="tok-num">0.01, epsilon * class="tok-num">0.999)   class="tok-comment"># décroissance exploration

print(fclass="tok-str">"Q-table entraînée, epsilon final : {epsilon:.4f}")

class="tok-comment"># ── class="tok-num">2. Deep Q-Network (CartPole) ─────────────────────────────────────────────
class DQN(nn.Module):
    def __init__(self, dim_obs, n_actions):
        super().__init__()
        self.reseau = nn.Sequential(
            nn.Linear(dim_obs, class="tok-num">128), nn.ReLU(),
            nn.Linear(class="tok-num">128, class="tok-num">128),    nn.ReLU(),
            nn.Linear(class="tok-num">128, n_actions)
        )
    def forward(self, x): return self.reseau(x)

env = gym.make(class="tok-str">"CartPole-v1")
dim_obs   = env.observation_space.shape[class="tok-num">0]
n_actions = env.action_space.n

q_reseau = DQN(dim_obs, n_actions)
q_cible  = DQN(dim_obs, n_actions)    class="tok-comment"># réseau cible gelé
q_cible.load_state_dict(q_reseau.state_dict())

optimiseur     = optim.Adam(q_reseau.parameters(), lr=class="tok-num">1e-3)
tampon_replay  = deque(maxlen=10_000)
gamma          = class="tok-num">0.99
epsilon        = class="tok-num">1.0
taille_lot     = class="tok-num">64
freq_maj_cible = class="tok-num">100
etapes         = class="tok-num">0

for episode in range(class="tok-num">500):
    s, _ = env.reset()
    recompense_totale = class="tok-num">0
    termine = False
    while not termine:
        class="tok-comment"># Action ε-greedy
        if np.random.random() < epsilon:
            a = env.action_space.sample()
        else:
            with torch.no_grad():
                a = q_reseau(torch.FloatTensor(s)).argmax().item()

        s_suiv, r, fin, tronque, _ = env.step(a)
        termine = fin or tronque
        tampon_replay.append((s, a, r, s_suiv, fin))
        s = s_suiv; recompense_totale += r; etapes += class="tok-num">1

        class="tok-comment"># Entraîner quand le tampon a assez d'échantillons
        if len(tampon_replay) >= taille_lot:
            lot = [tampon_replay[i] for i in np.random.choice(len(tampon_replay), taille_lot, replace=False)]
            S, A, R, S_suiv, D = map(np.array, zip(*lot))
            S_t = torch.FloatTensor(S); A_t = torch.LongTensor(A)
            R_t = torch.FloatTensor(R); S_suiv_t = torch.FloatTensor(S_suiv)
            D_t = torch.FloatTensor(D)

            with torch.no_grad():
                q_max_suiv = q_cible(S_suiv_t).max(class="tok-num">1)[class="tok-num">0]
                cible = R_t + gamma * q_max_suiv * (class="tok-num">1 - D_t)

            q_courant = q_reseau(S_t).gather(class="tok-num">1, A_t.unsqueeze(class="tok-num">1)).squeeze()
            perte = nn.functional.mse_loss(q_courant, cible)
            optimiseur.zero_grad(); perte.backward(); optimiseur.step()

        if etapes % freq_maj_cible == class="tok-num">0:
            q_cible.load_state_dict(q_reseau.state_dict())

    epsilon = max(class="tok-num">0.05, epsilon * class="tok-num">0.995)
    if episode % class="tok-num">50 == class="tok-num">0:
        print(fclass="tok-str">"Épisode {episode:4d} | récompense={recompense_totale:class="tok-num">5.0f} | ε={epsilon:.3f}")