# Omitted-Variable Bias Example
# Module 3 - Omitted-Variable Bias
# Dataset: WAGE1

# Omitted-Variable Bias Example
#
# Compare a short model and a model with controls, then discuss whether omitted variables may have moved the slope.

# %% Cell 2
from pathlib import Path

DATASET = "WAGE1.DTA"
VARIABLES = ["lwage", "educ", "exper", "tenure"]
DATA_FOLDER = "https://drive.google.com/drive/folders/1_STdcydIcst-opcbwOKRFzUXsgxQgBoS?usp=sharing"

if not Path(DATASET).exists():
    raise FileNotFoundError(
        "Dataset file not installed yet\n"
        f"Dataset: WAGE1\n"
        f"Variables needed: {', '.join(VARIABLES)}\n"
        f"Course data folder: {DATA_FOLDER}\n"
        "Admin upload instruction: upload the dataset in Admin -> Datasets and make it available to this notebook."
    )

# %% Cell 3
import pandas as pd
import numpy as np

df = pd.read_stata(DATASET)[VARIABLES].dropna()

def slope(columns):
    X = np.column_stack([np.ones(len(df)), df[columns].to_numpy()])
    beta = np.linalg.lstsq(X, df["lwage"].to_numpy(), rcond=None)[0]
    return dict(zip(["intercept", *columns], beta))

short = slope(["educ"])
controlled = slope(["educ", "exper", "tenure"])
print("Short model education coefficient:", round(short["educ"], 4))
print("Controlled education coefficient:", round(controlled["educ"], 4))
print("Difference:", round(short["educ"] - controlled["educ"], 4))

# Interpretation
#
# A coefficient difference is evidence that the omitted controls mattered in the sample. It is not, by itself, proof that all omitted-variable bias is solved.