أهمية الميزات والاختيار

from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np

class="tok-comment"># ── Synthetic tabular dataset ──────────────────────────────────────────────────
np.random.seed(class="tok-num">42)
n = class="tok-num">1000
X = pd.DataFrame({
    class="tok-str">"income":         np.random.normal(class="tok-num">50, class="tok-num">15, n),
    class="tok-str">"age":            np.random.randint(class="tok-num">18, class="tok-num">70, n),
    class="tok-str">"credit_score":   np.random.normal(class="tok-num">650, class="tok-num">80, n),
    class="tok-str">"loan_amount":    np.random.normal(class="tok-num">20, class="tok-num">8, n),
    class="tok-str">"employment_yrs": np.random.exponential(class="tok-num">5, n),
    class="tok-str">"num_accounts":   np.random.poisson(class="tok-num">3, n),
    class="tok-str">"random_noise":   np.random.randn(n),           class="tok-comment"># truly useless
    class="tok-str">"zip_code":       np.random.randint(class="tok-num">0, class="tok-num">10000, n), class="tok-comment"># high-cardinality noise
})
y = (
    class="tok-num">0.4 * (X[class="tok-str">"income"] > class="tok-num">55)
    + class="tok-num">0.3 * (X[class="tok-str">"credit_score"] > class="tok-num">660)
    + class="tok-num">0.2 * (X[class="tok-str">"age"] > class="tok-num">35)
    + class="tok-num">0.1 * np.random.rand(n)
) > class="tok-num">0.5

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=class="tok-num">0.2, random_state=class="tok-num">42)

class="tok-comment"># ── class="tok-num">1. Train Random Forest ─────────────────────────────────────────────────────
rf = RandomForestClassifier(n_estimators=class="tok-num">100, random_state=class="tok-num">42)
rf.fit(X_train, y_train)

class="tok-comment"># ── class="tok-num">2. Impurity (Gini) importance — fast, built-in ────────────────────────────
impurity_imp = pd.Series(rf.feature_importances_, index=X.columns)
print(class="tok-str">"Impurity importance:")
print(impurity_imp.sort_values(ascending=False).round(class="tok-num">3))
class="tok-comment"># WARNING: zip_code (high cardinality) may appear inflated here

class="tok-comment"># ── class="tok-num">3. Permutation importance — model-agnostic, honest ───────────────────────
perm = permutation_importance(
    rf, X_val, y_val,
    n_repeats=class="tok-num">10,          class="tok-comment"># shuffle class="tok-num">10 times, take mean ± std
    scoring=class="tok-str">"accuracy",
    random_state=class="tok-num">42,
    n_jobs=-class="tok-num">1
)
perm_imp = pd.DataFrame({
    class="tok-str">"mean": perm.importances_mean,
    class="tok-str">"std":  perm.importances_std,
}, index=X.columns).sort_values(class="tok-str">"mean", ascending=False)

print(class="tok-str">"\nPermutation importance:")
print(perm_imp.round(class="tok-num">3))
class="tok-comment"># random_noise and zip_code will be near zero or negative

class="tok-comment"># ── class="tok-num">4. Compare the two methods ────────────────────────────────────────────────
comparison = pd.DataFrame({
    class="tok-str">"impurity": impurity_imp,
    class="tok-str">"permutation": perm.importances_mean,
}).sort_values(class="tok-str">"permutation", ascending=False)
print(class="tok-str">"\nComparison (sorted by permutation):")
print(comparison.round(class="tok-num">3))

class="tok-comment"># ── class="tok-num">5. Feature selection using permutation importance ────────────────────────
selected = perm_imp[perm_imp[class="tok-str">"mean"] > class="tok-num">0.01].index.tolist()
print(fclass="tok-str">"\nSelected features ({len(selected)}): {selected}")