Sentiment analysis pipeline:
classic → transformer.

from datasets import load_dataset

# 25k train / 25k test, label 0 = negative, 1 = positive
ds = load_dataset("imdb")
train, test = ds["train"], ds["test"]

print(len(train), len(test))          # 25000 25000
print(train[0]["label"])             # 0
print(train[0]["text"][:120])        # "I rented I AM CURIOUS-YELLOW ..."

# class balance sanity check
from collections import Counter
print(Counter(train["label"]))     # Counter({0: 12500, 1: 12500})

import re

def clean(text):
    text = text.lower()
    text = re.sub(r"<br\s*/?>", " ", text)      # IMDb is full of <br/> tags
    text = re.sub(r"[^a-z\s']", " ", text)         # keep letters + apostrophes
    text = re.sub(r"\s+", " ", text).strip()        # collapse whitespace
    return text

print(clean("This movie was GREAT!!! <br/>Loved it."))
# -> "this movie was great loved it"

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score, f1_score, classification_report

X_train = [clean(t) for t in train["text"]]
X_test  = [clean(t) for t in test["text"]]
y_train, y_test = train["label"], test["label"]

clf = Pipeline([
    ("tfidf", TfidfVectorizer(
        ngram_range=(1, 2),   # unigrams + bigrams catch "not good"
        min_df=5,             # drop terms in < 5 docs (noise)
        max_features=50_000,
        sublinear_tf=True,      # 1 + log(tf), dampens repeats
        stop_words="english")),
    ("lr", LogisticRegression(C=10.0, max_iter=1000, n_jobs=-1)),
])

clf.fit(X_train, y_train)
pred = clf.predict(X_test)

print(f"accuracy: {accuracy_score(y_test, pred):.4f}")   # 0.8989
print(f"macro-F1: {f1_score(y_test, pred, average='macro'):.4f}")  # 0.8988

import numpy as np

vocab = np.array(clf.named_steps["tfidf"].get_feature_names_out())
weights = clf.named_steps["lr"].coef_[0]
order = np.argsort(weights)

print("most POSITIVE:", vocab[order[-8:]][::-1])
# ['excellent' 'perfect' 'wonderful' 'great' 'best' 'amazing' 'favorite' 'superb']
print("most NEGATIVE:", vocab[order[:8]])
# ['worst' 'waste' 'awful' 'boring' 'poorly' 'terrible' 'bad' 'disappointment']

from transformers import AutoTokenizer

ckpt = "distilbert-base-uncased"
tok = AutoTokenizer.from_pretrained(ckpt)

def tokenize(batch):
    # truncate to 256 tokens: most sentiment cues are early, and it
    # keeps fine-tuning fast. raise to 512 for the last accuracy point.
    return tok(batch["text"], truncation=True, max_length=256)

ds_tok = ds.map(tokenize, batched=True)
print(tok.tokenize("an unwatchable mess"))
# ['an', 'un', '##watch', '##able', 'mess']  <- subwords, no UNKs

import numpy as np
import evaluate
from transformers import (AutoModelForSequenceClassification,
                          TrainingArguments, Trainer,
                          DataCollatorWithPadding)

model = AutoModelForSequenceClassification.from_pretrained(ckpt, num_labels=2)
collator = DataCollatorWithPadding(tok)        # dynamic padding per batch
acc, f1 = evaluate.load("accuracy"), evaluate.load("f1")

def metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return {"accuracy": acc.compute(predictions=preds, references=labels)["accuracy"],
            "f1": f1.compute(predictions=preds, references=labels, average="macro")["f1"]}

args = TrainingArguments(
    output_dir="distilbert-imdb",
    learning_rate=2e-5,            # small LR: we're nudging, not retraining
    per_device_train_batch_size=16,
    num_train_epochs=2,              # 1-2 epochs is plenty for fine-tuning
    weight_decay=0.01,
    eval_strategy="epoch",
    fp16=True)                     # mixed precision on a GPU

trainer = Trainer(model=model, args=args,
                  train_dataset=ds_tok["train"],
                  eval_dataset=ds_tok["test"],
                  tokenizer=tok, data_collator=collator,
                  compute_metrics=metrics)

trainer.train()
print(trainer.evaluate())
# {'eval_accuracy': 0.9332, 'eval_f1': 0.9331, ...}

from sklearn.metrics import confusion_matrix, classification_report

logits = trainer.predict(ds_tok["test"]).predictions
bert_pred = logits.argmax(axis=-1)

print(confusion_matrix(y_test, bert_pred))
# [[11583   917]
#  [  753 11747]]
print(classification_report(y_test, bert_pred,
                            target_names=["neg", "pos"], digits=3))