نظرية المعلومات

import numpy as np
from scipy.special import xlogy    class="tok-comment"># handles class="tok-num">0 * log(class="tok-num">0) = class="tok-num">0 safely
from scipy.stats import entropy as scipy_entropy
import matplotlib.pyplot as plt

def entropy(p: np.ndarray, base: float = class="tok-num">2) -> float:
    class="tok-str">"""Shannon entropy H(p) in bits (base=class="tok-num">2) or nats (base=e)"""
    p = np.asarray(p, dtype=float)
    p = p[p > class="tok-num">0]                  class="tok-comment"># class="tok-num">0 * log(class="tok-num">0) = class="tok-num">0 by convention
    return -np.sum(p * np.log(p) / np.log(base))

def cross_entropy(p: np.ndarray, q: np.ndarray, eps: float = class="tok-num">1e-12) -> float:
    class="tok-str">"""H(p, q) = -sum p * log(q)"""
    p, q = np.asarray(p, dtype=float), np.asarray(q, dtype=float)
    return -np.sum(p * np.log(q + eps))

def kl_divergence(p: np.ndarray, q: np.ndarray, eps: float = class="tok-num">1e-12) -> float:
    class="tok-str">"""KL(p||q) — NOT symmetric"""
    p, q = np.asarray(p, dtype=float), np.asarray(q, dtype=float)
    mask = p > class="tok-num">0
    return np.sum(p[mask] * np.log((p[mask] + eps) / (q[mask] + eps)))

class="tok-comment"># ── class="tok-num">1. Entropy of various distributions ──────────────────────────────────────
print(class="tok-str">"Entropy examples (bits):")
print(fclass="tok-str">"  Fair coin [class="tok-num">0.5, class="tok-num">0.5]:        {entropy([class="tok-num">0.5, class="tok-num">0.5]):.4f}")  class="tok-comment"># class="tok-num">1.0 bit
print(fclass="tok-str">"  Biased coin [class="tok-num">0.99, class="tok-num">0.01]:    {entropy([class="tok-num">0.99, class="tok-num">0.01]):.4f}")  class="tok-comment"># ≈ class="tok-num">0.08 bits
print(fclass="tok-str">"  Uniform class="tok-num">8 classes:           {entropy([class="tok-num">1/class="tok-num">8]*class="tok-num">8):.4f}")  class="tok-comment"># class="tok-num">3.0 bits
print(fclass="tok-str">"  Certain [class="tok-num">1.0, class="tok-num">0.0]:          {entropy([class="tok-num">1.0, class="tok-num">0.0]):.4f}")  class="tok-comment"># class="tok-num">0.0 bits

class="tok-comment"># ── class="tok-num">2. Cross-entropy loss (classification) ────────────────────────────────────
class="tok-comment"># Ground truth (one-hot): cat
p_true = np.array([class="tok-num">1., class="tok-num">0., class="tok-num">0.])       class="tok-comment"># cat
class="tok-comment"># Model predictions:
q_good = np.array([class="tok-num">0.8, class="tok-num">0.1, class="tok-num">0.1])   class="tok-comment"># confident & correct
q_bad  = np.array([class="tok-num">0.1, class="tok-num">0.8, class="tok-num">0.1])   class="tok-comment"># confident & wrong
q_uncertain = np.array([class="tok-num">0.4, class="tok-num">0.3, class="tok-num">0.3])  class="tok-comment"># uncertain & correct lean

print(class="tok-str">"\nCross-entropy losses:")
print(fclass="tok-str">"  Good prediction:    {cross_entropy(p_true, q_good):.4f}")   class="tok-comment"># low
print(fclass="tok-str">"  Bad prediction:     {cross_entropy(p_true, q_bad):.4f}")    class="tok-comment"># high
print(fclass="tok-str">"  Uncertain but ok:   {cross_entropy(p_true, q_uncertain):.4f}")

class="tok-comment"># H(p,q) = H(p) + KL(p||q). Since H(p)=class="tok-num">0 for one-hot: CE = KL(p||q)
print(fclass="tok-str">"  KL(p_true||q_good) = {kl_divergence(p_true, q_good):.4f}")

class="tok-comment"># ── class="tok-num">3. KL divergence: asymmetry ───────────────────────────────────────────────
p = np.array([class="tok-num">0.6, class="tok-num">0.3, class="tok-num">0.1])
q = np.array([class="tok-num">0.3, class="tok-num">0.5, class="tok-num">0.2])
print(fclass="tok-str">"\nKL(p||q) = {kl_divergence(p,q):.4f}")
print(fclass="tok-str">"KL(q||p) = {kl_divergence(q,p):.4f}")  class="tok-comment"># different — not a distance

class="tok-comment"># ── class="tok-num">4. Information gain in decision trees ─────────────────────────────────────
def information_gain(parent, left, right):
    n = len(parent)
    n_l, n_r = len(left), len(right)
    h_p = scipy_entropy(np.bincount(parent) / n, base=class="tok-num">2)
    h_l = scipy_entropy(np.bincount(left)   / n_l, base=class="tok-num">2) if n_l > class="tok-num">0 else class="tok-num">0
    h_r = scipy_entropy(np.bincount(right)  / n_r, base=class="tok-num">2) if n_r > class="tok-num">0 else class="tok-num">0
    return h_p - (n_l/n * h_l + n_r/n * h_r)

class="tok-comment"># class="tok-num">10 samples: class="tok-num">6 class-class="tok-num">0, class="tok-num">4 class-class="tok-num">1. Split: left=[class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">1], right=[class="tok-num">0,class="tok-num">0,class="tok-num">1,class="tok-num">1,class="tok-num">1]
parent = np.array([class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">1,class="tok-num">1,class="tok-num">1,class="tok-num">1])
left   = np.array([class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">0,class="tok-num">1])
right  = np.array([class="tok-num">0,class="tok-num">0,class="tok-num">1,class="tok-num">1,class="tok-num">1])
print(fclass="tok-str">"\nInformation gain: {information_gain(parent, left, right):.4f} bits")