# R-Squared F Test
# Module 4 - R-Squared Form of the F Statistic
# Dataset: BWGHT

# R-Squared F Test
#
# Learning goal: run a real multiple-regression inference workflow and write a careful interpretation.

# Dataset check
# This cell confirms that the dataset file is available. If the file is missing, the notebook gives a safe warning rather than fake results.

# %% Cell 3
import os
DATASET = "BWGHT.DTA"
VARIABLES = ['bwght', 'cigs', 'faminc', 'motheduc', 'fatheduc']
if not os.path.exists(DATASET):
    raise FileNotFoundError(
        "Dataset file not installed yet\n"
        "Dataset: BWGHT\n"
        "Variables needed: bwght, cigs, faminc, motheduc, fatheduc\n"
        "Course data folder: https://drive.google.com/drive/folders/1_STdcydIcst-opcbwOKRFzUXsgxQgBoS?usp=sharing\n"
        "Admin upload instruction: upload the dataset in Admin -> Datasets, publish it, and make the file available to the notebook runner."
    )
print("Ready:", DATASET)

# Estimate the model
# Estimate OLS and inspect the coefficient, standard error, t statistic, and p-value.

# %% Cell 5
import pandas as pd
import statsmodels.api as sm
from scipy import stats

df = pd.read_stata(DATASET)[VARIABLES].dropna()
y = df["bwght"]
X = sm.add_constant(df[['cigs', 'faminc', 'motheduc', 'fatheduc']])
model = sm.OLS(y, X).fit()
print(model.summary())
print("Focus coefficient:", "motheduc", model.params["motheduc"])
print("Standard error:", model.bse["motheduc"])
print("t statistic:", model.tvalues["motheduc"])
print("two-sided p-value:", model.pvalues["motheduc"])

# Inference calculation
# Compute the lesson-specific inference object and connect it to the hypothesis.

# %% Cell 7
restricted = sm.OLS(y, sm.add_constant(df[["cigs", "faminc"]])).fit()
q = 2
r2_f = ((model.rsquared - restricted.rsquared) / q) / ((1 - model.rsquared) / model.df_resid)
p_value = 1 - stats.f.cdf(r2_f, q, model.df_resid)
print({"R2 form F": r2_f, "p_value": p_value})
print("Check: this shortcut is for nested exclusion restrictions with the same y and sample.")

# Check your understanding
# Write one sentence that distinguishes statistical significance from practical importance for this model.