# Chapter 10: Importing, Cleaning, Validating, and Exporting Data
# Fundamentals of Python for Financial Econometrics - Importing, Cleaning, Validating, and Exporting Data
# Dataset: Ceteris Lab teaching sample

# Importing, Cleaning, Validating, and Exporting Data
#
# **Opening question:** How can messy input be transformed without erasing the evidence of what was changed?

# %% Cell 2
import pandas as pd
from io import StringIO

raw = StringIO("date,value\n2026-01-01,2.1\n2026-02-01,NA\n2026-02-01,2.4")
df = pd.read_csv(raw, parse_dates=["date"], na_values=["NA"])
df["is_duplicate_date"] = df.duplicated("date", keep=False)
print(df.isna().sum().to_dict())
print(df["is_duplicate_date"].sum())

# **Interpretation check:** Interpretation. The missing-value count and duplicate flag describe separate quality concerns.

# %% Cell 4
def validate_series(frame):
    assert frame["date"].notna().all(), "dates must be present"
    assert frame["date"].is_monotonic_increasing, "dates must be sorted"
    assert frame["value"].dropna().between(-100, 100).all(), "value outside range"
    return True

print(validate_series(df.sort_values("date")))

# **Interpretation check:** Interpretation. The range is intentionally broad for illustration. Real thresholds should reflect the variable and units.

# %% Cell 6
import pandas as pd
from io import StringIO
raw = StringIO("date,value\n2026-01-01,2.1\n2026-02-01,NA\n2026-02-01,2.4")
df = pd.read_csv(raw, parse_dates=["date"], na_values=["NA"])
df["is_duplicate_date"] = df.duplicated("date", keep=False)
print(df.isna().sum().to_dict())

# Verified source output
#
# ```text
# {'date': 0, 'value': 1, 'is_duplicate_date': 0} 2
# ```
#
# ```text
# True
# ```
#
# ```text
# {'date': 0, 'value': 1, 'is_duplicate_date': 0}
# 2
# ```