02_data_understanding

import warnings
from pathlib import Path

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from datasets import load_dataset

warnings.filterwarnings("ignore")

pd.set_option('display.max_colwidth', None)                     # Não limitar a largura da coluna
pd.set_option('display.width', None)                            # Expandir a largura da exibição (evitar quebra de linha horizontal)

PROJECT_ROOT = Path.cwd().resolve().parents[0]
DATA_DIR = PROJECT_ROOT / "data"
RAW_DIR = DATA_DIR / "00-raw"
BRONZE_DIR = DATA_DIR / "01-bronze"
ARTIFACTS_DIR = PROJECT_ROOT / "artifacts"
FIGURES_DIR = ARTIFACTS_DIR / "figures"

RAW_DIR.mkdir(parents=True, exist_ok=True)
BRONZE_DIR.mkdir(parents=True, exist_ok=True)
FIGURES_DIR.mkdir(parents=True, exist_ok=True)

print("PROJECT_ROOT:", PROJECT_ROOT)
print("RAW_DIR:", RAW_DIR)
print("BRONZE_DIR:", BRONZE_DIR)
print("FIGURES_DIR:", FIGURES_DIR)

dataset = load_dataset("Itau-Unibanco/FAQ_BACEN")
dataset

train_df = dataset["train"].to_pandas()
test_df = dataset["test"].to_pandas()

df = pd.concat([train_df.assign(split="train"), test_df.assign(split="test")], ignore_index=True)

print("train shape:", train_df.shape)
print("test shape :", test_df.shape)
print("full shape :", df.shape)

df.head(5)

df.info()

null_summary = df.isna().sum().to_frame("qtd_nulos")
null_summary["pct_nulos"] = (null_summary["qtd_nulos"] / len(df) * 100).round(2)
null_summary

df = df.rename(
    columns={
        "questions": "question",
        "categories": "category",
        "answers": "answer",
    }
)

df.columns.tolist()

df[["question", "category", "answer", "split"]].sample(5, random_state=42)

n_classes = df["category"].nunique()
print("Número de categorias:", n_classes)

class_counts = (
    df["category"]
    .value_counts()
    .rename_axis("category")
    .reset_index(name="count")
)

class_counts.head(10)

class_counts.describe(include="all")

top_20 = class_counts.head(20)

plt.figure(figsize=(12, 8))
plt.barh(top_20["category"][::-1], top_20["count"][::-1])
plt.title("Top 20 categorias por frequência")
plt.xlabel("Quantidade")
plt.ylabel("Categoria")
plt.tight_layout()
plt.show()

df["question_num_chars"] = df["question"].astype(str).str.len()
df["question_num_words"] = df["question"].astype(str).str.split().str.len()

df["answer_num_chars"] = df["answer"].astype(str).str.len()
df["answer_num_words"] = df["answer"].astype(str).str.split().str.len()

df[[
    "question_num_chars",
    "question_num_words",
    "answer_num_chars",
    "answer_num_words"
]].describe()

plt.figure(figsize=(10, 5))
plt.hist(df["question_num_words"], bins=30)
plt.title("Distribuição do número de palavras nas perguntas")
plt.xlabel("Número de palavras")
plt.ylabel("Frequência")
plt.tight_layout()
plt.show()

plt.figure(figsize=(10, 5))
plt.hist(df["answer_num_words"], bins=30)
plt.title("Distribuição do número de palavras nas respostas")
plt.xlabel("Número de palavras")
plt.ylabel("Frequência")
plt.tight_layout()
plt.show()

print("Perguntas mais curtas:")
display(
    df.sort_values("question_num_words", ascending=True)[
        ["question", "category", "question_num_words"]
    ].head(10)
)

print("Perguntas mais longas:")
display(
    df.sort_values("question_num_words", ascending=False)[
        ["question", "category", "question_num_words"]
    ].head(10)
)

dup_question = df.duplicated(subset=["question"]).sum()
dup_question_category = df.duplicated(subset=["question", "category"]).sum()
dup_full = df.duplicated(subset=["question", "category", "answer"]).sum()

pd.DataFrame(
    {
        "criterio": ["question", "question + category", "question + category + answer"],
        "qtd_duplicados": [dup_question, dup_question_category, dup_full],
    }
)

split_summary = (
    df.groupby("split")
      .agg(
          n_registros=("question", "size"),
          n_categorias=("category", "nunique"),
          avg_question_words=("question_num_words", "mean"),
          avg_answer_words=("answer_num_words", "mean"),
      )
      .round(2)
)

split_summary

cross_tab = pd.crosstab(df["category"], df["split"])
cross_tab.head(20)

insights = [
    "O dataset possui estrutura adequada para classificação textual supervisionada.",
    "A variável 'category' será utilizada como target principal do MVP.",
    "O tamanho das perguntas tende a ser menor que o das respostas, o que favorece cenários de triagem por texto curto.",
    "Será necessário avaliar desbalanceamento entre categorias antes da modelagem.",
    "A coluna 'answer' pode ser usada depois em tarefas complementares, como recuperação de resposta ou QA.",
]

for i, insight in enumerate(insights, start=1):
    print(f"{i}. {insight}")

bronze_path_parquet = BRONZE_DIR / "faq_bacen_bronze.parquet"
bronze_path_csv = BRONZE_DIR / "faq_bacen_bronze.csv"

df.to_parquet(bronze_path_parquet, index=False)
df.to_csv(bronze_path_csv, index=False)

print("Arquivo salvo em:")
print(bronze_path_parquet)
print(bronze_path_csv)

#!uv pip install nbconvert -U -q
!jupyter nbconvert --to html --template-file my-template-html-v10.tpl 02_data_understanding.ipynb