التعلم المعزز

import numpy as np
import gymnasium as gym
from collections import deque
import torch, torch.nn as nn, torch.optim as optim

class="tok-comment"># ── class="tok-num">1. Tabular Q-Learning (FrozenLake) ───────────────────────────────────────
env = gym.make(class="tok-str">"FrozenLake-v1", is_slippery=False)
n_states, n_actions = env.observation_space.n, env.action_space.n

Q = np.zeros((n_states, n_actions))
alpha = class="tok-num">0.8          class="tok-comment"># learning rate
gamma = class="tok-num">0.95         class="tok-comment"># discount
epsilon = class="tok-num">1.0        class="tok-comment"># exploration rate

for episode in range(class="tok-num">2000):
    s, _ = env.reset()
    done = False
    while not done:
        class="tok-comment"># ε-greedy
        if np.random.random() < epsilon:
            a = env.action_space.sample()
        else:
            a = np.argmax(Q[s])

        s_next, r, terminated, truncated, _ = env.step(a)
        done = terminated or truncated

        class="tok-comment"># Q-learning update (Bellman target)
        td_target = r + gamma * np.max(Q[s_next]) * (class="tok-num">1 - terminated)
        Q[s, a] += alpha * (td_target - Q[s, a])
        s = s_next

    epsilon = max(class="tok-num">0.01, epsilon * class="tok-num">0.999)   class="tok-comment"># decay exploration

class="tok-comment"># Evaluate greedy policy
n_wins = class="tok-num">0
for _ in range(class="tok-num">100):
    s, _ = env.reset()
    done = False
    while not done:
        a = np.argmax(Q[s])
        s, r, terminated, truncated, _ = env.step(a)
        done = terminated or truncated
    if r == class="tok-num">1.0:      class="tok-comment"># reached the goal
        n_wins += class="tok-num">1
print(fclass="tok-str">"Win rate: {n_wins/class="tok-num">100:.class="tok-num">0%}")

class="tok-comment"># ── class="tok-num">2. Deep Q-Network (CartPole) ─────────────────────────────────────────────
class DQN(nn.Module):
    def __init__(self, obs_dim, n_actions):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(obs_dim, class="tok-num">128), nn.ReLU(),
            nn.Linear(class="tok-num">128, class="tok-num">128),    nn.ReLU(),
            nn.Linear(class="tok-num">128, n_actions)
        )
    def forward(self, x): return self.net(x)

env = gym.make(class="tok-str">"CartPole-v1")
obs_dim = env.observation_space.shape[class="tok-num">0]
n_actions = env.action_space.n

q_net    = DQN(obs_dim, n_actions)
q_target = DQN(obs_dim, n_actions)    class="tok-comment"># frozen target network
q_target.load_state_dict(q_net.state_dict())

optimizer  = optim.Adam(q_net.parameters(), lr=class="tok-num">1e-3)
replay     = deque(maxlen=10_000)     class="tok-comment"># experience replay buffer
gamma      = class="tok-num">0.99
epsilon    = class="tok-num">1.0
batch_size = class="tok-num">64
target_update_freq = class="tok-num">100
steps = class="tok-num">0

for episode in range(class="tok-num">500):
    s, _ = env.reset()
    total_reward = class="tok-num">0
    done = False
    while not done:
        class="tok-comment"># ε-greedy action
        if np.random.random() < epsilon:
            a = env.action_space.sample()
        else:
            with torch.no_grad():
                a = q_net(torch.FloatTensor(s)).argmax().item()

        s_next, r, terminated, truncated, _ = env.step(a)
        done = terminated or truncated
        replay.append((s, a, r, s_next, terminated))
        s = s_next; total_reward += r; steps += class="tok-num">1

        class="tok-comment"># Train when buffer has enough samples
        if len(replay) >= batch_size:
            batch = [replay[i] for i in np.random.choice(len(replay), batch_size, replace=False)]
            S, A, R, S_next, D = map(np.array, zip(*batch))
            S_t = torch.FloatTensor(S); A_t = torch.LongTensor(A)
            R_t = torch.FloatTensor(R); S_next_t = torch.FloatTensor(S_next)
            D_t = torch.FloatTensor(D)

            with torch.no_grad():
                max_next_q = q_target(S_next_t).max(class="tok-num">1)[class="tok-num">0]
                target = R_t + gamma * max_next_q * (class="tok-num">1 - D_t)

            current_q = q_net(S_t).gather(class="tok-num">1, A_t.unsqueeze(class="tok-num">1)).squeeze()
            loss = nn.functional.mse_loss(current_q, target)
            optimizer.zero_grad(); loss.backward(); optimizer.step()

        if steps % target_update_freq == class="tok-num">0:
            q_target.load_state_dict(q_net.state_dict())

    epsilon = max(class="tok-num">0.05, epsilon * class="tok-num">0.995)
    if episode % class="tok-num">50 == class="tok-num">0:
        print(fclass="tok-str">"Episode {episode:4d} | reward={total_reward:class="tok-num">5.0f} | ε={epsilon:.3f}")