# %% [markdown]
# # IEEE VIS Research Topic Evolution (1990–2024)
#
# This notebook implements the recommended visual analytics workflow to reveal
# major research topics in IEEE VIS publications and how they evolved over time.
#
# **Workflow:** Progressive Abstraction → Topic Mining → Interactive Steering →
# Cluster-Label-Distribute → Simplify for Trends

# %% [markdown]
# ## 0. Setup & Installation

# %%
# !pip install pandas numpy scikit-learn matplotlib seaborn nltk wordcloud adjustText

# %%
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path
import warnings
warnings.filterwarnings('ignore')

# NLP
import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

# Topic Modelling
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation
from sklearn.manifold import TSNE

nltk.download('stopwords', quiet=True)
nltk.download('wordnet', quiet=True)
nltk.download('omw-1.4', quiet=True)

print("Setup complete.")

# %% [markdown]
# ## 1. Data Loading & Exploration
#
# Expects a CSV file with at least columns: `Year`, `Title`, `Abstract`.

# %%
# === CONFIGURE THIS ===
DATA_PATH = "ieee_vis_papers.csv"  # <-- Adjust path to your dataset

# Load data
df = pd.read_csv(DATA_PATH)
df.columns = df.columns.str.strip().str.lower()

COLUMN_MAP = {
    'year': 'year',
    'title': 'title',
    'abstract': 'abstract',
}
df = df.rename(columns=COLUMN_MAP)

assert 'year' in df.columns
assert 'title' in df.columns
assert 'abstract' in df.columns

print(f"Dataset shape: {df.shape}")
print(f"Year range: {df['year'].min()} – {df['year'].max()}")
df.head(3)

# %%
# Drop rows with missing abstracts
before = len(df)
df = df.dropna(subset=['abstract']).copy()
df = df[df['abstract'].str.strip().str.len() > 50]
print(f"Dropped {before - len(df)} rows. Remaining: {len(df)}")

# %%
# Papers per year
fig, ax = plt.subplots(figsize=(12, 4))
df.groupby('year').size().plot(kind='bar', ax=ax, color='steelblue', width=0.8)
ax.set_title("Number of Papers per Year")
ax.set_xlabel("Year")
ax.set_ylabel("Count")
plt.tight_layout()
plt.show()

# %% [markdown]
# ## 2. Text Preprocessing (Pattern 1 – Progressive Abstraction, Level 1)

# %%
df['text'] = df['title'].fillna('') + ' ' + df['abstract'].fillna('')

CUSTOM_STOPWORDS = {
    'paper', 'propose', 'proposed', 'approach', 'method', 'methods',
    'result', 'results', 'show', 'shown', 'use', 'used', 'using',
    'present', 'presented', 'new', 'novel', 'technique', 'techniques',
    'based', 'provide', 'system', 'also', 'work', 'problem',
    'describe', 'described', 'demonstrate', 'demonstrated',
    'algorithm', 'algorithms', 'however', 'many', 'one', 'two',
    'first', 'second', 'allow', 'allows', 'well', 'can', 'may',
    'different', 'several', 'existing', 'within', 'number',
    'introduction', 'conclusion', 'abstract', 'study', 'studies',
    'example', 'examples', 'application', 'applications'
}

stop_words = set(stopwords.words('english')).union(CUSTOM_STOPWORDS)
lemmatizer = WordNetLemmatizer()

def preprocess_text(text):
    text = text.lower()
    text = re.sub(r'[^a-z\s]', ' ', text)
    tokens = text.split()
    tokens = [
        lemmatizer.lemmatize(t)
        for t in tokens
        if t not in stop_words and len(t) > 2
    ]
    return ' '.join(tokens)

df['processed_text'] = df['text'].apply(preprocess_text)
print("Preprocessing complete.")

# %% [markdown]
# ## 3. Topic Extraction (Pattern 11 – Topic Mining)
#
# NMF on TF-IDF features; intentionally over-segment (k=30).

# %%
N_TOPICS = 30
MAX_FEATURES = 5000
MIN_DF = 5
MAX_DF = 0.7
NGRAM_RANGE = (1, 2)

# %%
tfidf_vectorizer = TfidfVectorizer(
    max_features=MAX_FEATURES,
    min_df=MIN_DF, max_df=MAX_DF,
    ngram_range=NGRAM_RANGE,
    token_pattern=r'(?u)\b[a-z][a-z]+\b'
)
tfidf_matrix = tfidf_vectorizer.fit_transform(df['processed_text'])
feature_names = tfidf_vectorizer.get_feature_names_out()
print(f"TF-IDF matrix: {tfidf_matrix.shape[0]} docs × {tfidf_matrix.shape[1]} terms")

# %%
nmf_model = NMF(n_components=N_TOPICS, random_state=42, max_iter=500, init='nndsvda')
W = nmf_model.fit_transform(tfidf_matrix)
H = nmf_model.components_
print(f"NMF complete. Reconstruction error: {nmf_model.reconstruction_err_:.2f}")

# %%
N_TOP_TERMS = 12

def display_topics(model, feature_names, n_top=N_TOP_TERMS):
    topics = []
    for idx, topic_vec in enumerate(model.components_):
        top_indices = topic_vec.argsort()[:-n_top - 1:-1]
        top_terms = [feature_names[i] for i in top_indices]
        topics.append(top_terms)
        print(f"Topic {idx:2d}: {', '.join(top_terms)}")
    return topics

print("=" * 70)
print("TOP TERMS PER TOPIC")
print("=" * 70)
topic_terms = display_topics(nmf_model, feature_names)

# %%
df['dominant_topic'] = W.argmax(axis=1)
df['topic_weight'] = W.max(axis=1)

topic_distributions = pd.DataFrame(
    W, columns=[f"topic_{i}" for i in range(N_TOPICS)], index=df.index
)
print("\nPapers per topic:")
print(df['dominant_topic'].value_counts().sort_index())

# %% [markdown]
# ## 4. Interactive Topic Refinement (Pattern 12 – Model Steering)

# %%
from sklearn.metrics.pairwise import cosine_similarity

topic_similarity = cosine_similarity(H)

fig, ax = plt.subplots(figsize=(12, 10))
sns.heatmap(topic_similarity, annot=False, cmap='YlOrRd',
            xticklabels=range(N_TOPICS), yticklabels=range(N_TOPICS), ax=ax)
ax.set_title("Topic-Topic Cosine Similarity")
plt.tight_layout()
plt.show()

# %%
# Representative papers per topic
for t in range(N_TOPICS):
    topic_papers = df[df['dominant_topic'] == t].nlargest(3, 'topic_weight')
    print(f"\n--- Topic {t} ---")
    print(f"    Terms: {', '.join(topic_terms[t][:8])}")
    for _, row in topic_papers.iterrows():
        print(f"    [{row['year']}] {row['title'][:80]}")

# %% [markdown]
# ### 4a. Define Topic Merging & Labelling
#
# Edit MERGE_MAP, TOPIC_LABELS, and EXCLUDE_TOPICS based on inspection.

# %%
MERGE_MAP = {i: i for i in range(N_TOPICS)}
# MERGE_MAP[5] = 3    # Example: merge topic 5 into topic 3

EXCLUDE_TOPICS = []
# EXCLUDE_TOPICS = [0, 22]

TOPIC_LABELS = {i: f"Topic {i}" for i in range(N_TOPICS)}
# TOPIC_LABELS[0] = "Volume Rendering"
# TOPIC_LABELS[1] = "Graph & Network Visualization"
# ...

print("Merge map and labels defined. Adjust above as needed.")

# %%
df['merged_topic'] = df['dominant_topic'].map(MERGE_MAP)
df = df[~df['dominant_topic'].isin(EXCLUDE_TOPICS)].copy()

merged_groups = {}
for orig, merged in MERGE_MAP.items():
    if orig not in EXCLUDE_TOPICS:
        merged_groups.setdefault(merged, []).append(orig)

for group_id, members in merged_groups.items():
    member_cols = [f"topic_{m}" for m in members]
    existing_cols = [c for c in member_cols if c in topic_distributions.columns]
    if existing_cols:
        topic_distributions[f"merged_{group_id}"] = topic_distributions[existing_cols].sum(axis=1)

df['topic_label'] = df['merged_topic'].map(TOPIC_LABELS)

final_topics = sorted(df['merged_topic'].unique())
print(f"Number of final topics after merging: {len(final_topics)}")
for t in final_topics:
    count = (df['merged_topic'] == t).sum()
    print(f"  {TOPIC_LABELS.get(t, t):40s} ({count} papers)")

# %% [markdown]
# ## 5. Temporal Distribution (Pattern 4 – Cluster-Label-Distribute)

# %%
yearly_counts = df.groupby(['year', 'topic_label']).size().unstack(fill_value=0)
yearly_proportions = yearly_counts.div(yearly_counts.sum(axis=1), axis=0)
print(f"Yearly proportions table: {yearly_proportions.shape}")

# %%
fig, ax = plt.subplots(figsize=(16, 8))
yearly_proportions.plot.area(ax=ax, alpha=0.8, linewidth=0.5)
ax.set_title("Topic Proportions Over Time (Raw)", fontsize=14)
ax.set_xlabel("Year")
ax.set_ylabel("Proportion of Papers")
ax.legend(loc='upper left', bbox_to_anchor=(1.01, 1), fontsize=8)
ax.set_xlim(df['year'].min(), df['year'].max())
plt.tight_layout()
plt.show()

# %% [markdown]
# ## 6. Trend Smoothing (Pattern 5 – Simplify → Analyse → Restore)

# %%
WINDOW_SIZE = 5  # 5-year moving average

yearly_smoothed = yearly_proportions.rolling(
    window=WINDOW_SIZE, center=True, min_periods=2
).mean()
yearly_smoothed = yearly_smoothed.dropna()
print(f"Smoothed range: {yearly_smoothed.index.min()} – {yearly_smoothed.index.max()}")

# %%
fig, ax = plt.subplots(figsize=(16, 8))
yearly_smoothed.plot.area(ax=ax, alpha=0.85, linewidth=0.5)
ax.set_title(f"Topic Proportions ({WINDOW_SIZE}-Year Moving Average)", fontsize=14)
ax.set_xlabel("Year")
ax.set_ylabel("Proportion of Papers")
ax.legend(loc='upper left', bbox_to_anchor=(1.01, 1), fontsize=8)
plt.tight_layout()
plt.show()

# %%
# Small-multiple line charts
n_final_topics = len(yearly_smoothed.columns)
ncols = 4
nrows = int(np.ceil(n_final_topics / ncols))

fig, axes = plt.subplots(nrows, ncols, figsize=(16, 3*nrows), sharex=True, sharey=True)
axes_flat = axes.flatten()

for idx, topic_name in enumerate(yearly_smoothed.columns):
    ax = axes_flat[idx]
    ax.plot(yearly_smoothed.index, yearly_smoothed[topic_name], color='steelblue', lw=2)
    ax.fill_between(yearly_smoothed.index, yearly_smoothed[topic_name], alpha=0.3)
    ax.set_title(topic_name, fontsize=9, fontweight='bold')
    ax.grid(True, alpha=0.3)

for idx in range(n_final_topics, len(axes_flat)):
    axes_flat[idx].set_visible(False)

fig.suptitle(f"Individual Topic Trends ({WINDOW_SIZE}-Year Smoothing)", fontsize=14, y=1.01)
plt.tight_layout()
plt.show()

# %% [markdown]
# ## 7. Trend Classification & Summary

# %%
years = yearly_smoothed.index
n_years = len(years)
third = n_years // 3

early_period = yearly_smoothed.iloc[:third]
late_period = yearly_smoothed.iloc[-third:]

trend_summary = []
for topic_name in yearly_smoothed.columns:
    early_mean = early_period[topic_name].mean()
    late_mean = late_period[topic_name].mean()
    if early_mean > 0:
        rel_change = (late_mean - early_mean) / early_mean
    else:
        rel_change = 1.0 if late_mean > 0 else 0.0

    if rel_change > 0.3:
        trend = "📈 Emerging"
    elif rel_change < -0.3:
        trend = "📉 Declining"
    else:
        trend = "➡️ Stable"

    peak_year = yearly_smoothed[topic_name].idxmax()
    trend_summary.append({
        'Topic': topic_name,
        'Early Avg (%)': f"{early_mean*100:.1f}",
        'Late Avg (%)': f"{late_mean*100:.1f}",
        'Relative Change': f"{rel_change:+.0%}",
        'Peak Year': int(peak_year),
        'Trend': trend
    })

trend_df = pd.DataFrame(trend_summary).sort_values('Trend')
print("=" * 70)
print("TOPIC TREND SUMMARY")
print("=" * 70)
trend_df

# %% [markdown]
# ## 8. Detail-on-Demand (Pattern 5 – Restore)

# %%
DRILL_TOPIC = yearly_smoothed.columns[0]  # Change to topic of interest

print(f"Drilling into: '{DRILL_TOPIC}'")
topic_papers = df[df['topic_label'] == DRILL_TOPIC].sort_values('year')

for era_start in range(int(df['year'].min()), int(df['year'].max()) + 1, 5):
    era_end = era_start + 4
    era_papers = topic_papers[
        (topic_papers['year'] >= era_start) & (topic_papers['year'] <= era_end)
    ]
    if len(era_papers) > 0:
        print(f"\n--- {era_start}–{era_end} ({len(era_papers)} papers) ---")
        for _, row in era_papers.nlargest(3, 'topic_weight').iterrows():
            print(f"  [{row['year']}] {row['title'][:90]}")

# %% [markdown]
# ## 9. Export Results

# %%
output_dir = Path("output")
output_dir.mkdir(exist_ok=True)

df[['year', 'title', 'topic_label', 'topic_weight']].to_csv(
    output_dir / "papers_with_topics.csv", index=False)
yearly_smoothed.to_csv(output_dir / "yearly_topic_proportions_smoothed.csv")
trend_df.to_csv(output_dir / "trend_summary.csv", index=False)

print(f"Results saved to '{output_dir}/'")

# %% [markdown]
# ## 10. (Optional) 2-D Topic Map

# %%
if N_TOPICS > 5:
    perplexity = min(5, N_TOPICS - 1)
    tsne = TSNE(n_components=2, random_state=42, perplexity=perplexity)
    topic_coords = tsne.fit_transform(H)

    fig, ax = plt.subplots(figsize=(10, 8))
    ax.scatter(topic_coords[:, 0], topic_coords[:, 1],
               s=100, c=range(N_TOPICS), cmap='tab20', alpha=0.8, edgecolors='black')
    for i in range(N_TOPICS):
        if i not in EXCLUDE_TOPICS:
            ax.annotate(TOPIC_LABELS.get(i, f"T{i}"),
                       (topic_coords[i, 0], topic_coords[i, 1]),
                       fontsize=7, ha='center', va='bottom')
    ax.set_title("Topic Map (t-SNE of topic-term vectors)")
    plt.tight_layout()
    plt.show()

{
 "cells": [
  {"cell_type": "markdown", "source": ["# IEEE VIS Research Topic Evolution (1990–2024)\n", ...]},
  {"cell_type": "code", "source": ["# !pip install pandas numpy scikit-learn ..."]},
  {"cell_type": "code", "source": ["import pandas as pd\n", "import numpy as np\n", ...]},
  {"cell_type": "markdown", "source": ["## 1. Data Loading & Exploration\n", ...]},
  {"cell_type": "code", "source": ["DATA_PATH = \"ieee_vis_papers.csv\"\n", ...]},
  {"cell_type": "code", "source": ["# Drop rows with missing abstracts\n", ...]},
  {"cell_type": "code", "source": ["# Papers per year\n", ...]},
  {"cell_type": "markdown", "source": ["## 2. Text Preprocessing\n", ...]},
  {"cell_type": "code", "source": ["df['text'] = ...\n", "CUSTOM_STOPWORDS = {...}\n", ...]},
  {"cell_type": "markdown", "source": ["## 3. Topic Extraction (NMF)\n", ...]},
  {"cell_type": "code", "source": ["N_TOPICS = 30\n", ...]},
  {"cell_type": "code", "source": ["tfidf_vectorizer = TfidfVectorizer(...)\n", ...]},
  {"cell_type": "code", "source": ["nmf_model = NMF(...)\n", ...]},
  {"cell_type": "code", "source": ["# Display top terms per topic\n", ...]},
  {"cell_type": "code", "source": ["# Assign dominant topic\n", ...]},
  {"cell_type": "markdown", "source": ["## 4. Interactive Topic Refinement\n", ...]},
  {"cell_type": "code", "source": ["# Cosine similarity heatmap\n", ...]},
  {"cell_type": "code", "source": ["# Representative papers\n", ...]},
  {"cell_type": "markdown", "source": ["### 4a. Merging & Labelling\n", ...]},
  {"cell_type": "code", "source": ["MERGE_MAP = {...}\n", "EXCLUDE_TOPICS = [...]\n", ...]},
  {"cell_type": "code", "source": ["# Apply merging\n", ...]},
  {"cell_type": "markdown", "source": ["## 5. Temporal Distribution\n", ...]},
  {"cell_type": "code", "source": ["yearly_counts = ...\n", ...]},
  {"cell_type": "code", "source": ["# Raw stacked area chart\n", ...]},
  {"cell_type": "markdown", "source": ["## 6. Trend Smoothing\n", ...]},
  {"cell_type": "code", "source": ["WINDOW_SIZE = 5\n", ...]},
  {"cell_type": "code", "source": ["# Smoothed stacked area chart\n", ...]},
  {"cell_type": "code", "source": ["# Small-multiple line charts\n", ...]},
  {"cell_type": "markdown", "source": ["## 7. Trend Classification\n", ...]},
  {"cell_type": "code", "source": ["# Classify: Emerging / Declining / Stable\n", ...]},
  {"cell_type": "code", "source": ["# Highlight top trends\n", ...]},
  {"cell_type": "markdown", "source": ["## 8. Detail-on-Demand\n", ...]},
  {"cell_type": "code", "source": ["DRILL_TOPIC = ...\n", ...]},
  {"cell_type": "markdown", "source": ["## 9. Export Results\n", ...]},
  {"cell_type": "code", "source": ["output_dir = Path('output')\n", ...]},
  {"cell_type": "markdown", "source": ["## 10. (Optional) Topic Map\n", ...]},
  {"cell_type": "code", "source": ["# t-SNE of topic-term vectors\n", ...]},
  {"cell_type": "markdown", "source": ["---\n", "## Summary\n", ...]}
 ],
 "metadata": {
  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
  "language_info": {"name": "python", "version": "3.10.12"}
 },
 "nbformat": 4, "nbformat_minor": 4
}