معالجة اللغة الطبيعية: خط أنابيب تصنيف النصوص

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import ComplementNB
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.metrics import classification_report
import numpy as np

class="tok-comment"># ── Sample text data ───────────────────────────────────────────────────
corpus = [
    class="tok-str">"machine learning algorithms data science python",
    class="tok-str">"neural network deep learning pytorch tensorflow",
    class="tok-str">"natural language processing text classification bert",
    class="tok-str">"computer vision image recognition convolutional",
    class="tok-str">"reinforcement learning reward policy agent",
    class="tok-str">"data preprocessing feature engineering pipeline",
] * class="tok-num">40   class="tok-comment"># class="tok-num">240 samples, class="tok-num">6 classes
labels = list(range(class="tok-num">6)) * class="tok-num">40
X_text = corpus
y = np.array(labels)
X_train, X_test, y_train, y_test = train_test_split(
    X_text, y, test_size=class="tok-num">0.2, stratify=y, random_state=class="tok-num">42)
class="tok-comment"># indices for sentence-transformer section
train_idx = np.arange(len(X_train))
test_idx  = np.arange(len(X_test))

class="tok-comment"># ── Baseline: TF-IDF + Logistic Regression ────────────────────────
pipe_lr = Pipeline([
    (class="tok-str">'tfidf', TfidfVectorizer(
        ngram_range=(class="tok-num">1,class="tok-num">2),
        max_features=100_000,
        sublinear_tf=True,          class="tok-comment"># log(class="tok-num">1+tf) dampens high frequencies
        strip_accents=class="tok-str">'unicode',
        analyzer=class="tok-str">'word',
        token_pattern=rclass="tok-str">'\w{class="tok-num">2,}',  class="tok-comment"># ignore single-char tokens
        min_df=class="tok-num">2,                   class="tok-comment"># ignore very rare words
    )),
    (class="tok-str">'clf', LogisticRegression(C=class="tok-num">1.0, max_iter=class="tok-num">1000, class_weight=class="tok-str">'balanced')),
])

class="tok-comment"># ── Alternative: TF-IDF + LinearSVC (fast, great for text) ────────
pipe_svm = Pipeline([
    (class="tok-str">'tfidf', TfidfVectorizer(ngram_range=(class="tok-num">1,class="tok-num">2), max_features=100_000, sublinear_tf=True)),
    (class="tok-str">'clf', LinearSVC(C=class="tok-num">0.5, class_weight=class="tok-str">'balanced', max_iter=class="tok-num">2000)),
])

class="tok-comment"># ── Evaluate both with cross-validation ───────────────────────────
for name, pipe in [(class="tok-str">'LR', pipe_lr), (class="tok-str">'SVM', pipe_svm)]:
    scores = cross_val_score(pipe, X_text, y, cv=class="tok-num">5, scoring=class="tok-str">'f1_macro', n_jobs=-class="tok-num">1)
    print(fclass="tok-str">"{name}: macro-F1 = {scores.mean():.3f} ± {scores.std():.3f}")

class="tok-comment"># ── Modern approach: sentence embeddings ──────────────────────────
class="tok-comment"># pip install sentence-transformers
from sentence_transformers import SentenceTransformer
from sklearn.linear_model import LogisticRegression

encoder = SentenceTransformer(class="tok-str">'all-MiniLM-L6-v2')
X_emb = encoder.encode(X_text, batch_size=class="tok-num">256, show_progress_bar=True)
clf = LogisticRegression(max_iter=class="tok-num">1000).fit(X_emb[train_idx], y[train_idx])
print(fclass="tok-str">"Sentence-BERT accuracy: {clf.score(X_emb[test_idx], y[test_idx]):.3f}")