05_modeling__tranformer_text_classifier

import warnings
from pathlib import Path
import json
import os
import random

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import torch
from datasets import Dataset

from transformers import (
    AutoTokenizer,
    AutoModelForSequenceClassification,
    TrainingArguments,
    Trainer,
    DataCollatorWithPadding,
)

from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
from sklearn.metrics import classification_report

warnings.filterwarnings("ignore")

SEED = 42

random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
if torch.cuda.is_available():
    torch.cuda.manual_seed_all(SEED)

pd.set_option("display.max_columns", 200)
pd.set_option("display.max_rows", 200)
pd.set_option("display.max_colwidth", 200)

PROJECT_ROOT = Path.cwd().resolve().parents[0]
DATA_DIR = PROJECT_ROOT / "data"

GOLD_DIR = DATA_DIR / "03-gold"
TRANSFORMER_MODELS_DIR = PROJECT_ROOT / "models" / "transformer"
REPORTS_DIR = PROJECT_ROOT / "artifacts" / "reports"
FIGURES_DIR = PROJECT_ROOT / "artifacts" / "figures"

TRANSFORMER_MODELS_DIR.mkdir(parents=True, exist_ok=True)
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
FIGURES_DIR.mkdir(parents=True, exist_ok=True)

print("PROJECT_ROOT         :", PROJECT_ROOT)
print("GOLD_DIR             :", GOLD_DIR)
print("TRANSFORMER_MODELS_DIR:", TRANSFORMER_MODELS_DIR)
print("REPORTS_DIR          :", REPORTS_DIR)
print("FIGURES_DIR          :", FIGURES_DIR)
print("CUDA disponível?     :", torch.cuda.is_available())

train_path = GOLD_DIR / "faq_bacen_gold_train.csv"
valid_path = GOLD_DIR / "faq_bacen_gold_valid.csv"
test_path = GOLD_DIR / "faq_bacen_gold_test.csv"
label_mapping_path = GOLD_DIR / "label_mapping.csv"

baseline_metrics_path = REPORTS_DIR / "baseline_test_metrics.csv"

train_gold = pd.read_csv(train_path)
valid_gold = pd.read_csv(valid_path)
test_gold = pd.read_csv(test_path)
label_mapping_df = pd.read_csv(label_mapping_path).sort_values("label_id")

baseline_test_metrics_df = pd.read_csv(baseline_metrics_path)

print("train_gold:", train_gold.shape)
print("valid_gold:", valid_gold.shape)
print("test_gold :", test_gold.shape)
print("label_mapping_df:", label_mapping_df.shape)
print("baseline_test_metrics_df:", baseline_test_metrics_df.shape)

label_ids = label_mapping_df["label_id"].tolist()
label_names = label_mapping_df["category_final"].tolist()

id_to_label = dict(zip(label_mapping_df["label_id"], label_mapping_df["category_final"]))
label_to_id = dict(zip(label_mapping_df["category_final"], label_mapping_df["label_id"]))

N_LABELS = len(label_ids)

print("Número de labels:", N_LABELS)
print("Primeiras labels:", label_names[:10])

TEXT_COL = "question_clean_light"
TARGET_COL = "label_id"

MODEL_CHECKPOINT = "neuralmind/bert-base-portuguese-cased"
MAX_LENGTH = 128

print("TEXT_COL         :", TEXT_COL)
print("TARGET_COL       :", TARGET_COL)
print("MODEL_CHECKPOINT :", MODEL_CHECKPOINT)
print("MAX_LENGTH       :", MAX_LENGTH)

train_gold[[TEXT_COL, "category_final", TARGET_COL]].sample(5, random_state=42)

train_df = train_gold[[TEXT_COL, TARGET_COL, "category_final"]].copy()
valid_df = valid_gold[[TEXT_COL, TARGET_COL, "category_final"]].copy()
test_df = test_gold[[TEXT_COL, TARGET_COL, "category_final"]].copy()

train_df = train_df.rename(columns={TEXT_COL: "text", TARGET_COL: "label"})
valid_df = valid_df.rename(columns={TEXT_COL: "text", TARGET_COL: "label"})
test_df = test_df.rename(columns={TEXT_COL: "text", TARGET_COL: "label"})

train_hf = Dataset.from_pandas(train_df[["text", "label"]], preserve_index=False)
valid_hf = Dataset.from_pandas(valid_df[["text", "label"]], preserve_index=False)
test_hf = Dataset.from_pandas(test_df[["text", "label"]], preserve_index=False)

train_hf, valid_hf, test_hf

tokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT)
tokenizer

def tokenize_batch(batch):
    return tokenizer(
        batch["text"],
        truncation=True,
        max_length=MAX_LENGTH,
    )

train_tokenized = train_hf.map(tokenize_batch, batched=True)
valid_tokenized = valid_hf.map(tokenize_batch, batched=True)
test_tokenized = test_hf.map(tokenize_batch, batched=True)

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

train_tokenized[0]

model = AutoModelForSequenceClassification.from_pretrained(
    MODEL_CHECKPOINT,
    num_labels=N_LABELS,
    id2label=id_to_label,
    label2id=label_to_id,
)

def compute_transformer_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)

    return {
        "accuracy": accuracy_score(labels, preds),
        "macro_f1": f1_score(labels, preds, average="macro", zero_division=0),
        "weighted_f1": f1_score(labels, preds, average="weighted", zero_division=0),
        "macro_precision": precision_score(labels, preds, average="macro", zero_division=0),
        "macro_recall": recall_score(labels, preds, average="macro", zero_division=0),
    }

OUTPUT_DIR = TRANSFORMER_MODELS_DIR / "bertimbau_faq_bacen"

training_args = TrainingArguments(
    output_dir=str(OUTPUT_DIR),
    eval_strategy="epoch",
    save_strategy="epoch",
    logging_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
    load_best_model_at_end=True,
    metric_for_best_model="macro_f1",
    greater_is_better=True,
    save_total_limit=2,
    report_to="none",
    seed=SEED,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_tokenized,
    eval_dataset=valid_tokenized,
    processing_class=tokenizer,
    #tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_transformer_metrics,
)

train_result = trainer.train()

valid_metrics = trainer.evaluate(valid_tokenized)
valid_metrics

test_output = trainer.predict(test_tokenized)

test_logits = test_output.predictions
test_pred = np.argmax(test_logits, axis=-1)
y_test = test_df["label"].values

transformer_test_metrics = {
    "accuracy": accuracy_score(y_test, test_pred),
    "macro_f1": f1_score(y_test, test_pred, average="macro", zero_division=0),
    "weighted_f1": f1_score(y_test, test_pred, average="weighted", zero_division=0),
    "macro_precision": precision_score(y_test, test_pred, average="macro", zero_division=0),
    "macro_recall": recall_score(y_test, test_pred, average="macro", zero_division=0),
}

transformer_test_metrics

transformer_test_metrics_df = pd.DataFrame([transformer_test_metrics])
transformer_test_metrics_df.style.format(
    {
        "accuracy": "{:.4f}",
        "macro_f1": "{:.4f}",
        "weighted_f1": "{:.4f}",
        "macro_precision": "{:.4f}",
        "macro_recall": "{:.4f}",
    }
)

test_report_dict = classification_report(
    y_test,
    test_pred,
    labels=label_ids,
    target_names=label_names,
    output_dict=True,
    zero_division=0,
)

transformer_test_report_df = pd.DataFrame(test_report_dict).T
transformer_test_class_report = transformer_test_report_df.loc[label_names].copy()
transformer_test_class_report = transformer_test_class_report.sort_values(
    by="f1-score",
    ascending=False
)

transformer_test_class_report.head(20)

transformer_test_predictions_df = test_gold.copy()
transformer_test_predictions_df["y_true"] = y_test
transformer_test_predictions_df["y_pred"] = test_pred
transformer_test_predictions_df["true_label"] = transformer_test_predictions_df["y_true"].map(id_to_label)
transformer_test_predictions_df["pred_label"] = transformer_test_predictions_df["y_pred"].map(id_to_label)
transformer_test_predictions_df["is_error"] = (
    transformer_test_predictions_df["y_true"] != transformer_test_predictions_df["y_pred"]
)

transformer_test_predictions_df[[
    "question_raw",
    "true_label",
    "pred_label",
    "is_error"
]].head(15)

baseline_metrics = baseline_test_metrics_df.iloc[0].to_dict()

comparison_df = pd.DataFrame(
    [
        {"model": "baseline_tfidf_logreg", **baseline_metrics},
        {"model": "transformer_bertimbau", **transformer_test_metrics},
    ]
)

comparison_df

comparison_df.style.format(
    {
        "accuracy": "{:.4f}",
        "macro_f1": "{:.4f}",
        "weighted_f1": "{:.4f}",
        "macro_precision": "{:.4f}",
        "macro_recall": "{:.4f}",
    }
)

plt.figure(figsize=(8, 4))
plt.bar(comparison_df["model"], comparison_df["macro_f1"])
plt.title("Comparação de modelos — Macro F1 no teste")
plt.ylabel("Macro F1")
plt.xlabel("Modelo")
plt.tight_layout()
plt.show()

trainer.save_model(str(OUTPUT_DIR))
tokenizer.save_pretrained(str(OUTPUT_DIR))

transformer_metrics_path = REPORTS_DIR / "transformer_test_metrics.csv"
transformer_report_path = REPORTS_DIR / "transformer_test_classification_report.csv"
transformer_predictions_path = REPORTS_DIR / "transformer_test_predictions.csv"
comparison_path = REPORTS_DIR / "baseline_vs_transformer_comparison.csv"
transformer_metadata_path = TRANSFORMER_MODELS_DIR / "transformer_metadata.json"

transformer_test_metrics_df.to_csv(transformer_metrics_path, index=False)
transformer_test_class_report.to_csv(transformer_report_path)
transformer_test_predictions_df.to_csv(transformer_predictions_path, index=False)
comparison_df.to_csv(comparison_path, index=False)

transformer_metadata = {
    "project": "nlp-triagem-inteligente-financeira",
    "model_family": "Transformer",
    "checkpoint": MODEL_CHECKPOINT,
    "text_column": TEXT_COL,
    "target_column": TARGET_COL,
    "max_length": MAX_LENGTH,
    "n_labels": int(N_LABELS),
    "train_shape": list(train_gold.shape),
    "valid_shape": list(valid_gold.shape),
    "test_shape": list(test_gold.shape),
    "valid_metrics": valid_metrics,
    "test_metrics": transformer_test_metrics,
}

with open(transformer_metadata_path, "w", encoding="utf-8") as f:
    json.dump(transformer_metadata, f, ensure_ascii=False, indent=2)

print("Artefatos do transformer salvos com sucesso.")
print(transformer_metrics_path)
print(transformer_report_path)
print(transformer_predictions_path)
print(comparison_path)
print(transformer_metadata_path)

#!uv pip install nbconvert -U -q
!jupyter nbconvert --to html --template-file my-template-html-v10.tpl 05_modeling__tranformer_text_classifier.ipynb