# LM Test with CRIME1
# Module 5 - The Lagrange Multiplier Test
# Dataset: CRIME1

# LM Test with CRIME1
#
# Learning goal: Compute the n-R-squared LM statistic and compare with an F test.

# Dataset check
# This cell confirms the dataset file is available. Simulation notebooks mount a harmless CSV so public file delivery is still validated.

# %% Cell 3
import os
DATASET = "CRIME1.csv"
VARIABLES = ["narr86","pcnv","avgsen","tottime","ptime86","qemp86"]
if not os.path.exists(DATASET):
    raise FileNotFoundError(
        "Dataset file not installed yet\n"
        "Dataset: CRIME1\n"
        "Variables needed: narr86, pcnv, avgsen, tottime, ptime86, qemp86\n"
        "Course data folder: https://drive.google.com/drive/folders/1_STdcydIcst-opcbwOKRFzUXsgxQgBoS?usp=sharing\n"
        "Admin upload instruction: upload the dataset in Admin -> Datasets, publish it, and make the file available to the notebook runner."
    )
print("Ready:", DATASET)

# Run the lab
# Run the code, inspect the table or graph, and connect the result to the formula in the lesson.

# %% Cell 5
import pandas as pd
import statsmodels.api as sm
from scipy.stats import chi2

crime = pd.read_csv("CRIME1.csv")
cols = ["narr86", "pcnv", "avgsen", "tottime", "ptime86", "qemp86"]
crime = crime.dropna(subset=cols)
X_r = sm.add_constant(crime[["pcnv", "ptime86", "qemp86"]])
restricted = sm.OLS(crime["narr86"], X_r).fit()
crime["u_restricted"] = restricted.resid
X_aux = sm.add_constant(crime[["pcnv", "ptime86", "qemp86", "avgsen", "tottime"]])
auxiliary = sm.OLS(crime["u_restricted"], X_aux).fit()
LM = len(crime) * auxiliary.rsquared
q = 2
p_value = 1 - chi2.cdf(LM, q)
print("LM statistic:", LM)
print("p-value:", p_value)
print("Auxiliary R-squared:", auxiliary.rsquared)

# Formula explanation
# Explain the probability limit, asymptotic approximation, standard-error pattern, or LM statistic in words. Do not treat output as automatic causal evidence.

# Short exercise
# Change one sample size, regressor, or restriction. Write two sentences: what changed mechanically, and what assumption still matters?
#
# Check your understanding
# Does increasing n fix nonnormality, endogeneity, heteroskedasticity, or omitted variables? Explain.