06_evaluation_business_interpretation

from pathlib import Path
import json

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

pd.set_option("display.max_columns", 200)
pd.set_option("display.max_rows", 300)
pd.set_option("display.max_colwidth", 200)

PROJECT_ROOT = Path.cwd().resolve().parents[0]
REPORTS_DIR = PROJECT_ROOT / "artifacts" / "reports"
MODELS_DIR = PROJECT_ROOT / "models"
BASELINE_MODELS_DIR = MODELS_DIR / "baseline"
TRANSFORMER_MODELS_DIR = MODELS_DIR / "transformer"
FIGURES_DIR = PROJECT_ROOT / "artifacts" / "figures"

print("PROJECT_ROOT          :", PROJECT_ROOT)
print("REPORTS_DIR           :", REPORTS_DIR)
print("BASELINE_MODELS_DIR   :", BASELINE_MODELS_DIR)
print("TRANSFORMER_MODELS_DIR:", TRANSFORMER_MODELS_DIR)
print("FIGURES_DIR           :", FIGURES_DIR)

baseline_metrics_path = REPORTS_DIR / "baseline_test_metrics.csv"
baseline_report_path = REPORTS_DIR / "baseline_test_classification_report.csv"
baseline_predictions_path = REPORTS_DIR / "baseline_test_predictions.csv"

transformer_metrics_path = REPORTS_DIR / "transformer_test_metrics.csv"
transformer_report_path = REPORTS_DIR / "transformer_test_classification_report.csv"
transformer_predictions_path = REPORTS_DIR / "transformer_test_predictions.csv"

comparison_path = REPORTS_DIR / "baseline_vs_transformer_comparison.csv"

baseline_metrics_df = pd.read_csv(baseline_metrics_path)
baseline_report_df = pd.read_csv(baseline_report_path, index_col=0)
baseline_predictions_df = pd.read_csv(baseline_predictions_path)

transformer_metrics_df = pd.read_csv(transformer_metrics_path)
transformer_report_df = pd.read_csv(transformer_report_path, index_col=0)
transformer_predictions_df = pd.read_csv(transformer_predictions_path)

if comparison_path.exists():
    comparison_df = pd.read_csv(comparison_path)
else:
    comparison_df = pd.DataFrame()

print("baseline_metrics_df        :", baseline_metrics_df.shape)
print("baseline_report_df         :", baseline_report_df.shape)
print("baseline_predictions_df    :", baseline_predictions_df.shape)
print("transformer_metrics_df     :", transformer_metrics_df.shape)
print("transformer_report_df      :", transformer_report_df.shape)
print("transformer_predictions_df :", transformer_predictions_df.shape)
print("comparison_df              :", comparison_df.shape)

baseline_metrics = baseline_metrics_df.iloc[0].to_dict()
transformer_metrics = transformer_metrics_df.iloc[0].to_dict()

comparison_df = pd.DataFrame(
    [
        {"model": "baseline_tfidf_logreg", **baseline_metrics},
        {"model": "transformer_bertimbau", **transformer_metrics},
    ]
)

comparison_df

comparison_df.style.format(
    {
        "accuracy": "{:.4f}",
        "macro_f1": "{:.4f}",
        "weighted_f1": "{:.4f}",
        "macro_precision": "{:.4f}",
        "macro_recall": "{:.4f}",
    }
)

delta_df = pd.DataFrame(
    [
        {
            "metric": metric,
            "baseline": float(baseline_metrics[metric]),
            "transformer": float(transformer_metrics[metric]),
            "delta_transformer_minus_baseline": float(transformer_metrics[metric]) - float(baseline_metrics[metric]),
        }
        for metric in ["accuracy", "macro_f1", "weighted_f1", "macro_precision", "macro_recall"]
    ]
)

delta_df

delta_df.style.format(
    {
        "baseline": "{:.4f}",
        "transformer": "{:.4f}",
        "delta_transformer_minus_baseline": "{:+.4f}",
    }
)

plt.figure(figsize=(9, 5))
plt.bar(delta_df["metric"], delta_df["delta_transformer_minus_baseline"])
plt.axhline(0, linewidth=1)
plt.title("Delta do transformer em relação ao baseline")
plt.xlabel("Métrica")
plt.ylabel("Transformer - Baseline")
plt.tight_layout()
plt.show()

transformer_pred_counts = (
    transformer_predictions_df["pred_label"]
    .value_counts()
    .rename_axis("pred_label")
    .reset_index(name="count")
)

transformer_pred_counts.head(20)

n_unique_transformer_preds = transformer_predictions_df["pred_label"].nunique()
n_unique_baseline_preds = baseline_predictions_df["pred_label"].nunique()

pd.DataFrame(
    [
        {"model": "baseline_tfidf_logreg", "n_unique_predicted_labels": n_unique_baseline_preds},
        {"model": "transformer_bertimbau", "n_unique_predicted_labels": n_unique_transformer_preds},
    ]
)

top_transformer_pred = transformer_pred_counts.iloc[0].to_dict()
top_transformer_pred

baseline_class_df = baseline_report_df.copy()
transformer_class_df = transformer_report_df.copy()

for df_ in [baseline_class_df, transformer_class_df]:
    for col in ["precision", "recall", "f1-score", "support"]:
        if col in df_.columns:
            df_[col] = pd.to_numeric(df_[col], errors="coerce")

summary_rows = {"accuracy", "macro avg", "weighted avg"}

baseline_classes_only = baseline_class_df.loc[~baseline_class_df.index.isin(summary_rows)].copy()
transformer_classes_only = transformer_class_df.loc[~transformer_class_df.index.isin(summary_rows)].copy()

class_comparison_df = baseline_classes_only[["precision", "recall", "f1-score", "support"]].rename(
    columns={
        "precision": "baseline_precision",
        "recall": "baseline_recall",
        "f1-score": "baseline_f1",
        "support": "support",
    }
).join(
    transformer_classes_only[["precision", "recall", "f1-score"]].rename(
        columns={
            "precision": "transformer_precision",
            "recall": "transformer_recall",
            "f1-score": "transformer_f1",
        }
    ),
    how="inner"
)

class_comparison_df["delta_f1"] = class_comparison_df["transformer_f1"] - class_comparison_df["baseline_f1"]
class_comparison_df["delta_recall"] = class_comparison_df["transformer_recall"] - class_comparison_df["baseline_recall"]

class_comparison_df.head(20)

class_comparison_df.style.format(
    {
        "baseline_precision": "{:.4f}",
        "baseline_recall": "{:.4f}",
        "baseline_f1": "{:.4f}",
        "transformer_precision": "{:.4f}",
        "transformer_recall": "{:.4f}",
        "transformer_f1": "{:.4f}",
        "delta_f1": "{:+.4f}",
        "delta_recall": "{:+.4f}",
    }
)

top_gain_classes = class_comparison_df.sort_values(by="delta_f1", ascending=False).head(15).copy()
top_loss_classes = class_comparison_df.sort_values(by="delta_f1", ascending=True).head(15).copy()

top_gain_classes[["baseline_f1", "transformer_f1", "delta_f1", "support"]]

top_loss_classes[["baseline_f1", "transformer_f1", "delta_f1", "support"]]

plot_loss_df = class_comparison_df.sort_values(by="delta_f1", ascending=True).head(12)

plt.figure(figsize=(14, 6))
plt.barh(plot_loss_df.index[::-1], plot_loss_df["delta_f1"][::-1])
plt.title("Top 12 classes com maior perda de F1 no transformer")
plt.xlabel("Delta F1")
plt.ylabel("Classe")
plt.tight_layout()
plt.show()

baseline_errors = baseline_predictions_df[baseline_predictions_df["is_error"] == True].copy()
transformer_errors = transformer_predictions_df[transformer_predictions_df["is_error"] == True].copy()

error_summary_df = pd.DataFrame(
    [
        {"model": "baseline_tfidf_logreg", "n_errors": baseline_errors.shape[0]},
        {"model": "transformer_bertimbau", "n_errors": transformer_errors.shape[0]},
    ]
)

error_summary_df

plt.figure(figsize=(10, 5))
plt.bar(error_summary_df["model"], error_summary_df["n_errors"])
plt.title("Quantidade total de erros no conjunto de teste")
plt.ylabel("Número de erros")
plt.xlabel("Modelo")
plt.tight_layout()
plt.show()

baseline_top_confusions = (
    baseline_errors.groupby(["true_label", "pred_label"])
    .size()
    .reset_index(name="count")
    .query("true_label != pred_label")
    .sort_values("count", ascending=False)
    .head(15)
)

transformer_top_confusions = (
    transformer_errors.groupby(["true_label", "pred_label"])
    .size()
    .reset_index(name="count")
    .query("true_label != pred_label")
    .sort_values("count", ascending=False)
    .head(15)
)

baseline_top_confusions.head(10)

transformer_top_confusions.head(10)

baseline_errors[["question_raw", "true_label", "pred_label"]].head(10)

transformer_errors[["question_raw", "true_label", "pred_label"]].head(10)

macro_f1_delta = float(transformer_metrics["macro_f1"]) - float(baseline_metrics["macro_f1"])
accuracy_delta = float(transformer_metrics["accuracy"]) - float(baseline_metrics["accuracy"])
weighted_f1_delta = float(transformer_metrics["weighted_f1"]) - float(baseline_metrics["weighted_f1"])

business_interpretation_df = pd.DataFrame(
    [
        ["Ganho global", "Macro F1", macro_f1_delta],
        ["Ganho global", "Accuracy", accuracy_delta],
        ["Ganho global", "Weighted F1", weighted_f1_delta],
    ],
    columns=["dimensao", "metrica", "delta"]
)

business_interpretation_df

recommendation_df = pd.DataFrame(
    [
        [
            "Baseline TF-IDF + Logistic Regression",
            "Melhor desempenho observado no teste; maior simplicidade operacional; menor custo",
            "Pode perder capacidade em classes semanticamente muito próximas",
            "Deployment inicial / MVP operacional / fallback robusto",
        ],
        [
            "Transformer",
            "Maior potencial contextual em formulações futuras",
            "Não validado com sucesso na formulação atual; colapso para classe dominante",
            "Trilha de evolução experimental, não candidato principal no estado atual",
        ],
    ],
    columns=["opcao", "forca_principal", "limitacao_principal", "cenario_indicado"]
)

recommendation_df

delta_path = REPORTS_DIR / "baseline_vs_transformer_metric_deltas.csv"
class_comparison_path = REPORTS_DIR / "baseline_vs_transformer_class_comparison.csv"
recommendation_path = REPORTS_DIR / "deployment_recommendation_table.csv"
error_summary_path = REPORTS_DIR / "model_error_summary.csv"
transformer_pred_distribution_path = REPORTS_DIR / "transformer_prediction_distribution.csv"

delta_df.to_csv(delta_path, index=False)
class_comparison_df.to_csv(class_comparison_path)
recommendation_df.to_csv(recommendation_path, index=False)
error_summary_df.to_csv(error_summary_path, index=False)
transformer_pred_counts.to_csv(transformer_pred_distribution_path, index=False)

print("Arquivos consolidados salvos com sucesso:")
print(delta_path)
print(class_comparison_path)
print(recommendation_path)
print(error_summary_path)
print(transformer_pred_distribution_path)

#!uv pip install nbconvert -U -q
!jupyter nbconvert --to html --template-file my-template-html-v10.tpl 06_evaluation_business_interpretation.ipynb