# Chapter 51: Natural-Language Processing and Transformers for Economics
# Fundamentals of Python for Financial Econometrics - Natural-Language Processing and Transformers for Economics
# Dataset: Ceteris Lab teaching sample

# Natural-Language Processing and Transformers for Economics
#
# **Opening question:** How can text be converted into numerical representations while preserving enough context to classify or retrieve meaning?

# %% Cell 2
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

docs = ["inflation rose after energy prices increased", "energy costs pushed inflation higher", "the football match ended in a draw"]
X = TfidfVectorizer().fit_transform(docs)
print(cosine_similarity(X[0], X).round(3).tolist()[0])

# **Interpretation check:** Interpretation. The two inflation documents share weighted vocabulary, while the unrelated sports sentence has zero overlap in this tiny corpus.

# %% Cell 4
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

texts = ["flood warning issued", "sunny picnic today", "earthquake reported", "great concert tonight"]
labels = [1, 0, 1, 0]
pipe = Pipeline([("tfidf", TfidfVectorizer()), ("model", LogisticRegression())]).fit(texts, labels)
print(pipe.predict(["storm warning tonight"]).tolist())

# **Interpretation check:** Interpretation. The example demonstrates workflow, not reliable disaster detection. Four training texts are far too few for deployment.

# %% Cell 6
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
X = TfidfVectorizer().fit_transform(docs)
print(cosine_similarity(X[0], X).round(3).tolist()[0])
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression

# Verified source output
#
# ```text
# [1.0, 0.25, 0.0]
# ```
#
# ```text
# [0]
# ```