# F Tests for Exclusion Restrictions
# Module 4 - F Tests for Exclusion Restrictions
# Dataset: BWGHT

# F Tests for Exclusion Restrictions
#
# Learning goal: run a real multiple-regression inference workflow and write a careful interpretation.

# Dataset check
# This cell confirms that the dataset file is available. If the file is missing, the notebook gives a safe warning rather than fake results.

# %% Cell 3
import os
DATASET = "BWGHT.DTA"
VARIABLES = ['bwght', 'cigs', 'faminc', 'motheduc', 'fatheduc']
if not os.path.exists(DATASET):
    raise FileNotFoundError(
        "Dataset file not installed yet\n"
        "Dataset: BWGHT\n"
        "Variables needed: bwght, cigs, faminc, motheduc, fatheduc\n"
        "Course data folder: https://drive.google.com/drive/folders/1_STdcydIcst-opcbwOKRFzUXsgxQgBoS?usp=sharing\n"
        "Admin upload instruction: upload the dataset in Admin -> Datasets, publish it, and make the file available to the notebook runner."
    )
print("Ready:", DATASET)

# Estimate the model
# Estimate OLS and inspect the coefficient, standard error, t statistic, and p-value.

# %% Cell 5
import pandas as pd
import statsmodels.api as sm
from scipy import stats

df = pd.read_stata(DATASET)[VARIABLES].dropna()
y = df["bwght"]
X = sm.add_constant(df[['cigs', 'faminc', 'motheduc', 'fatheduc']])
model = sm.OLS(y, X).fit()
print(model.summary())
print("Focus coefficient:", "motheduc", model.params["motheduc"])
print("Standard error:", model.bse["motheduc"])
print("t statistic:", model.tvalues["motheduc"])
print("two-sided p-value:", model.pvalues["motheduc"])

# Inference calculation
# Compute the lesson-specific inference object and connect it to the hypothesis.

# %% Cell 7
restricted = sm.OLS(y, sm.add_constant(df[["cigs", "faminc"]])).fit()
q = 2
f_stat = ((restricted.ssr - model.ssr) / q) / (model.ssr / model.df_resid)
p_value = 1 - stats.f.cdf(f_stat, q, model.df_resid)
print({"F": f_stat, "p_value": p_value, "df_num": q, "df_den": model.df_resid})
print(model.f_test("motheduc = 0, fatheduc = 0"))

# Check your understanding
# Write one sentence that distinguishes statistical significance from practical importance for this model.