# WAGE1 Multiple Regression
# Module 3 - Multiple Regression in Python
# Dataset: WAGE1

# WAGE1 Multiple Regression
#
# Estimate log wage on education, experience, and tenure. This notebook does not include saved regression output; run the cells after installing the dataset.

# %% Cell 2
from pathlib import Path

DATASET = "WAGE1.DTA"
VARIABLES = ["wage", "lwage", "educ", "exper", "tenure"]
DATA_FOLDER = "https://drive.google.com/drive/folders/1_STdcydIcst-opcbwOKRFzUXsgxQgBoS?usp=sharing"

if not Path(DATASET).exists():
    raise FileNotFoundError(
        "Dataset file not installed yet\n"
        f"Dataset: WAGE1\n"
        f"Variables needed: {', '.join(VARIABLES)}\n"
        f"Course data folder: {DATA_FOLDER}\n"
        "Admin upload instruction: upload the dataset in Admin -> Datasets and make it available to this notebook."
    )

print("Dataset found:", DATASET)

# %% Cell 3
import pandas as pd
import numpy as np

df = pd.read_stata(DATASET)[VARIABLES].dropna()
print(df.head())
print("Rows used:", len(df))

# %% Cell 4
y = df["lwage"].to_numpy()
X = np.column_stack([np.ones(len(df)), df[["educ", "exper", "tenure"]].to_numpy()])
names = ["intercept", "educ", "exper", "tenure"]
beta = np.linalg.lstsq(X, y, rcond=None)[0]
fitted = X @ beta
residuals = y - fitted
r_squared = 1 - np.sum(residuals ** 2) / np.sum((y - y.mean()) ** 2)

for name, value in zip(names, beta):
    print(f"{name}: {value:.4f}")
print("R-squared:", round(r_squared, 4))

# Interpretation
#
# Read the education coefficient as the predicted change in log wage for one more year of education, holding experience and tenure fixed. This is a conditional association unless the regression assumptions are justified.