# White Test
# Module 8 - The White Test
# Dataset: MODULE8_HOUSING_VARIANCE_SYNTHETIC

# White Test
#
# Module 8 notebook lab. This notebook uses original Ceteris Lab teaching data and does not report real empirical findings.

# Learning goal
# Use full and fitted-value White diagnostics.
#
# Dataset: MODULE8_HOUSING_VARIANCE_SYNTHETIC. Variables: house_id, price, log_price, lotsize, sqrft, bedrooms.

# %% Cell 3
import pandas as pd
import statsmodels.api as sm
from statsmodels.stats.diagnostic import het_white

df = pd.read_csv("/data/module-8/module8_housing_variance_synthetic.csv")
X = sm.add_constant(df[["sqrft", "lotsize", "bedrooms"]])
model = sm.OLS(df["price"], X).fit()
lm, lm_pvalue, fvalue, f_pvalue = het_white(model.resid, X)
print({"White LM": round(lm, 4), "LM p-value": round(lm_pvalue, 4), "F": round(fvalue, 4), "F p-value": round(f_pvalue, 4)})

import pandas as pd
import statsmodels.api as sm

df = pd.read_csv("/data/module-8/module8_housing_variance_synthetic.csv")
X = sm.add_constant(df[["sqrft", "lotsize", "bedrooms"]])
model = sm.OLS(df["price"], X).fit()
aux_x = pd.DataFrame({"fitted": model.fittedvalues, "fitted_sq": model.fittedvalues ** 2})
aux = sm.OLS(model.resid ** 2, sm.add_constant(aux_x)).fit()
print(aux.summary().tables[1])
print("Special White LM = nR^2:", round(len(df) * aux.rsquared, 4))

# Reflection
# Write two sentences: one sentence explaining what the diagnostic or robust result says, and one sentence explaining a limitation or next step.