# Histograms, Skewness, and Transformations
# Module 5 - Histograms, Normality, and Transformations
# Dataset: WAGE1

# Histograms, Skewness, and Transformations
#
# Learning goal: Compare residual histograms for wage and log wage.

# Dataset check
# This cell confirms the dataset file is available. Simulation notebooks mount a harmless CSV so public file delivery is still validated.

# %% Cell 3
import os
DATASET = "WAGE1.csv"
VARIABLES = ["wage","educ","exper","tenure"]
if not os.path.exists(DATASET):
    raise FileNotFoundError(
        "Dataset file not installed yet\n"
        "Dataset: WAGE1\n"
        "Variables needed: wage, educ, exper, tenure\n"
        "Course data folder: https://drive.google.com/drive/folders/1_STdcydIcst-opcbwOKRFzUXsgxQgBoS?usp=sharing\n"
        "Admin upload instruction: upload the dataset in Admin -> Datasets, publish it, and make the file available to the notebook runner."
    )
print("Ready:", DATASET)

# Run the lab
# Run the code, inspect the table or graph, and connect the result to the formula in the lesson.

# %% Cell 5
import pandas as pd
import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt

data = pd.read_csv("WAGE1.csv")
data = data.dropna(subset=["wage", "educ", "exper", "tenure"])
X = sm.add_constant(data[["educ", "exper", "tenure"]])
model_level = sm.OLS(data["wage"], X).fit()
data["log_wage"] = np.log(data["wage"])
model_log = sm.OLS(data["log_wage"], X).fit()
fig, axes = plt.subplots(1, 2, figsize=(10, 4))
axes[0].hist(model_level.resid, bins=30)
axes[0].set_title("Residuals: wage model")
axes[0].set_xlabel("residual")
axes[1].hist(model_log.resid, bins=30)
axes[1].set_title("Residuals: log(wage) model")
axes[1].set_xlabel("residual")
plt.tight_layout()
print("Level residual skewness:", pd.Series(model_level.resid).skew())
print("Log residual skewness:", pd.Series(model_log.resid).skew())

# Formula explanation
# Explain the probability limit, asymptotic approximation, standard-error pattern, or LM statistic in words. Do not treat output as automatic causal evidence.

# Short exercise
# Change one sample size, regressor, or restriction. Write two sentences: what changed mechanically, and what assumption still matters?
#
# Check your understanding
# Does increasing n fix nonnormality, endogeneity, heteroskedasticity, or omitted variables? Explain.