# Residuals and Fitted Values
# Module 3 - Fitted Values and Residuals
# Dataset: WAGE1

# Residuals and Fitted Values
#
# Create fitted log wages and residuals from a multiple-regression model.

# %% Cell 2
from pathlib import Path

DATASET = "WAGE1.DTA"
VARIABLES = ["lwage", "educ", "exper", "tenure"]
DATA_FOLDER = "https://drive.google.com/drive/folders/1_STdcydIcst-opcbwOKRFzUXsgxQgBoS?usp=sharing"

if not Path(DATASET).exists():
    raise FileNotFoundError(
        "Dataset file not installed yet\n"
        f"Dataset: WAGE1\n"
        f"Variables needed: {', '.join(VARIABLES)}\n"
        f"Course data folder: {DATA_FOLDER}\n"
        "Admin upload instruction: upload the dataset in Admin -> Datasets and make it available to this notebook."
    )

# %% Cell 3
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

df = pd.read_stata(DATASET)[VARIABLES].dropna()
y = df["lwage"].to_numpy()
X = np.column_stack([np.ones(len(df)), df[["educ", "exper", "tenure"]].to_numpy()])
beta = np.linalg.lstsq(X, y, rcond=None)[0]
df["fitted_lwage"] = X @ beta
df["residual"] = y - df["fitted_lwage"]

print(df[["lwage", "fitted_lwage", "residual"]].head())
print("Residual mean:", round(df["residual"].mean(), 8))

# %% Cell 4
plt.scatter(df["fitted_lwage"], df["residual"], alpha=0.6)
plt.axhline(0, linestyle="--")
plt.xlabel("Fitted log wage")
plt.ylabel("Residual")
plt.title("Residuals versus fitted values")
plt.show()

# Interpretation
#
# A residual is the part of the outcome not fitted by the included variables. The plot helps students look for patterns the model may have missed.