feat: finalize empirical ordinal tier rankings, fix entity collisions, and dual-mode dashboard

app.py

@@ -1,9 +1,9 @@
 """
 app.py
 HIFFI Artist Intelligence - Multi-Platform Executive Dashboard
-Clean dark-themed design with multi-platform evidence tiers, Spotify & Instagram telemetry,
+Dual-Engine Architecture:
-ChartMasters record sales (CSPC/EAS), breakout velocity index, deep-dive scouting dossier,
+1. HIFFI Hip-Hop 500: Multi-platform evidence tiers, YouTube/Spotify/IG telemetry, BVI, CPI engine.
-and experimental Commercial Power Index (CPI) simulation engine with sensitivity analysis.
+2. Global Master Catalog (17k): Empirical registry verification (CSPC, RIAA, Last.fm) with ordinal tier ranking.
 """
 
 from pathlib import Path
@@ -12,8 +12,15 @@ import plotly.express as px
 import plotly.graph_objects as go
 import streamlit as st
 
+try:
     from core.cpi_engine import CommercialPowerEngine
+except ImportError:
+    CommercialPowerEngine = None
+
+try:
     from exporters.excel_workbook import generate_executive_sales_workbook
+except ImportError:
+    generate_executive_sales_workbook = None
 
 st.set_page_config(
     page_title="HIFFI | Multi-Platform Intelligence",
@@ -22,8 +29,9 @@ st.set_page_config(
     initial_sidebar_state="expanded",
 )
 
-DATA_PATH = Path("data/hiffi_multiplatform_500.csv")
+PATH_500 = Path("data/hiffi_multiplatform_500.csv")
-FALLBACK_DATA_PATH = Path("enriched_artists.csv")
+PATH_17K = Path("data/master_artists_intelligence_17k_final.csv")
+FALLBACK_PATH = Path("enriched_artists.csv")
 
 # Global CSS Styling
 st.markdown(
@@ -36,8 +44,6 @@ st.markdown(
         padding-right: 2.5rem;
         max-width: 100% !important;
     }
-
-    /* KPI Cards */
     .kpi-box {
         background: #111827;
         border: 1px solid #1f2937;
@@ -62,8 +68,6 @@ st.markdown(
         font-size: 0.75rem;
         color: #6b7280;
     }
-
-    /* Telemetry Coverage Banner */
     .coverage-card {
         background: #0f172a;
         border: 1px solid #1e293b;
@@ -75,8 +79,6 @@ st.markdown(
         align-items: center;
         flex-wrap: wrap;
     }
-
-    /* Simulation Micro-Cards */
     .sim-kpi-card {
         background: #0f172a;
         border: 1px solid #334155;
@@ -95,8 +97,6 @@ st.markdown(
         font-weight: 800;
         margin-top: 2px;
     }
-
-    /* Podium Cards */
     .podium-tile {
         background: #111827;
         border-radius: 12px;
@@ -107,8 +107,6 @@ st.markdown(
     .podium-gold { border-top: 4px solid #f59e0b; }
     .podium-silver { border-top: 4px solid #94a3b8; }
     .podium-bronze { border-top: 4px solid #f97316; }
-
-    /* Badges */
     .badge {
         display: inline-block;
         padding: 3px 8px;
@@ -121,7 +119,6 @@ st.markdown(
     .badge-tier2 { background: rgba(56, 189, 248, 0.2); color: #38bdf8; }
     .badge-tier1 { background: rgba(245, 158, 11, 0.2); color: #f59e0b; }
     .badge-dormant { background: rgba(239, 68, 68, 0.2); color: #ef4444; }
-
     .custom-divider {
         height: 1px;
         background: #1f2937;
@@ -134,13 +131,14 @@ st.markdown(
 
 
 @st.cache_data(ttl=60)
-def load_data():
+def load_hiffi_500():
-    target = DATA_PATH if DATA_PATH.exists() else FALLBACK_DATA_PATH
+    target = PATH_500 if PATH_500.exists() else FALLBACK_PATH
     if not target.exists():
         return pd.DataFrame()
 
     df = pd.read_csv(target)
-
+    if "name" not in df.columns and "artist_name" in df.columns:
+        df["name"] = df["artist_name"]
     if "hiffi_xp_published_rank" not in df.columns:
         df["hiffi_xp_published_rank"] = df.get("hiffi_xp_rank", df.get("hiffi_rank", df.index + 1))
     if "hiffi_xp_score" not in df.columns:
@@ -152,38 +150,24 @@ def load_data():
         df["platform_evidence_tier"] = has_yt + has_sp + has_ig
 
     numeric_cols = [
-        "hiffi_xp_published_rank",
+        "hiffi_xp_published_rank", "hiffi_xp_rank", "hiffi_xp_score", "production_hiffi_rank",
-        "hiffi_xp_rank",
+        "production_hiffi_score", "platform_evidence_tier", "breakout_velocity_index",
-        "hiffi_xp_score",
+        "subscriber_count", "daily_view_velocity", "momentum_30d", "spotify_monthly_listeners",
-        "production_hiffi_rank",
+        "spotify_daily_change", "instagram_followers", "confidence_score", "breakout_rank",
-        "production_hiffi_score",
+        "chartmasters_sales_rank", "cohort_sales_rank", "pure_album_sales",
-        "platform_evidence_tier",
+        "physical_singles_sales", "digital_singles_sales", "streaming_eas", "total_eas_cspc",
-        "breakout_velocity_index",
+        "riaa_certified_album_units", "riaa_certified_singles_units"
-        "subscriber_count",
-        "daily_view_velocity",
-        "momentum_30d",
-        "spotify_monthly_listeners",
-        "spotify_daily_change",
-        "instagram_followers",
-        "confidence_score",
-        "breakout_rank",
-        "chartmasters_sales_rank",
-        "cohort_sales_rank",
-        "pure_album_sales",
-        "physical_singles_sales",
-        "digital_singles_sales",
-        "streaming_eas",
-        "total_eas_cspc",
-        "riaa_certified_album_units",
-        "riaa_certified_singles_units",
     ]
     for col in numeric_cols:
         if col in df.columns:
             df[col] = pd.to_numeric(df[col], errors="coerce")
 
-    # Commercial Power Engine
+    if CommercialPowerEngine is not None:
-    engine = CommercialPowerEngine()
+        try:
-    df["commercial_power_index"] = engine.calculate_cpi(df)
+            eng = CommercialPowerEngine()
+            df["commercial_power_index"] = eng.calculate_cpi(df)
+        except Exception:
+            df["commercial_power_index"] = None
 
     if "subgenre" not in df.columns:
         def infer_subgenre(r):
@@ -200,32 +184,128 @@ def load_data():
     return df
 
 
-df = load_data()
+@st.cache_data(ttl=60)
-engine = CommercialPowerEngine()
+def load_master_17k():
+    if not PATH_17K.exists():
+        return pd.DataFrame()
+    df = pd.read_csv(PATH_17K)
+    if "name" not in df.columns and "artist_name" in df.columns:
+        df["name"] = df["artist_name"]
+    return df
+
+
+df_500 = load_hiffi_500()
+df_17k = load_master_17k()
+engine = CommercialPowerEngine() if CommercialPowerEngine is not None else None
 
 
 def get_provenance_badge(status: str) -> str:
-    """Format status values into readable provenance labels."""
     s = str(status).strip()
-    if s == "VERIFIED_CSPC":
+    if "CSPC" in s or s == "VERIFIED_CSPC":
         return "✅ VERIFIED (CSPC)"
-    elif s == "VERIFIED_RIAA":
+    elif "RIAA" in s or "COMMERCIAL_CORE" in s:
         return "🏅 VERIFIED (RIAA)"
-    elif s == "ESTIMATED_SEA":
+    elif "LASTFM" in s:
+        return "📻 VERIFIED (Last.fm)"
+    elif "ESTIMATED_SEA" in s:
         return "📊 ESTIMATED (SEA)"
-    return "⏳ PENDING_INGESTION"
+    return "⏳ UNAUDITED / PENDING"
 
 
 # ----------------- Sidebar Controls -----------------
 with st.sidebar:
-    st.markdown("### ⚡ HIFFI Multi-Platform")
+    st.markdown("### ⚡ HIFFI Intelligence Suite")
-    st.caption("YouTube • Spotify • Instagram • Sales Telemetry")
+    st.caption("Hip-Hop 500 & Master Catalog Intelligence")
     st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
 
-    if df.empty:
+    # Dataset Mode Switcher
-        st.error("No dataset found. Run pipeline to generate master data.")
+    catalog_source = st.selectbox(
+        "🗂️ Active Data Scope:",
+        ["⚡ Hip-Hop 500 (Executive Telemetry)", "🌐 Global Master Catalog (17,000+ Acts)"],
+        index=0
+    )
+    st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
+
+# =====================================================================
+# MODULE 1: GLOBAL MASTER CATALOG (17,000+ ACTS)
+# =====================================================================
+if "Global Master Catalog" in catalog_source:
+    with st.sidebar:
+        st.markdown("##### 🔍 17k Catalog Filters")
+        if df_17k.empty:
+            st.error("Master 17k dataset not found at data/master_artists_intelligence_17k_final.csv")
             st.stop()
 
+        tier_list = ["All Tiers"] + sorted(df_17k["catalog_intelligence_tier"].dropna().unique().tolist())
+        sel_tier = st.selectbox("Intelligence Tier:", tier_list, index=0)
+        search_17k = st.text_input("Search Artist Name:", placeholder="e.g. Eminem, The Roots...")
+
+    filt_17k = df_17k.copy()
+    if sel_tier != "All Tiers":
+        filt_17k = filt_17k[filt_17k["catalog_intelligence_tier"] == sel_tier]
+    if search_17k:
+        filt_17k = filt_17k[filt_17k["name"].str.contains(search_17k, case=False, na=False)]
+
+    head1, head2 = st.columns([3, 1])
+    with head1:
+        st.markdown("<h2 style='margin-bottom: 2px;'>🌐 Global Master Artist Catalog (17k)</h2>", unsafe_allow_html=True)
+        st.markdown("<p style='color: #9ca3af; font-size: 0.92rem;'>Empirical registry validation via CSPC sales, RIAA certifications, and verified Last.fm global scrobbles.</p>", unsafe_allow_html=True)
+    with head2:
+        st.markdown(f"<div style='text-align: right; color: #6b7280; font-size: 0.8rem; padding-top: 10px;'>Filtered: <b>{len(filt_17k):,} / {len(df_17k):,}</b> acts</div>", unsafe_allow_html=True)
+
+    # 17k Macro KPIs
+    mk1, mk2, mk3, mk4, mk5 = st.columns(5)
+    with mk1:
+        st.markdown(f'<div class="kpi-box"><div class="kpi-title">Universe Size</div><div class="kpi-value">{len(df_17k):,}</div><div class="kpi-subtitle">Total Cataloged</div></div>', unsafe_allow_html=True)
+    with mk2:
+        ver_cnt = df_17k["catalog_rank"].notna().sum()
+        st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #10b981;">Verified Entities</div><div class="kpi-value" style="color: #10b981;">{ver_cnt:,}</div><div class="kpi-subtitle">Registry Authenticated</div></div>', unsafe_allow_html=True)
+    with mk3:
+        t1_cnt = (df_17k["catalog_intelligence_tier"] == "TIER_1_GLOBAL_SUPERSTAR_CSPC").sum()
+        st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #f59e0b;">Tier 1 Superstars</div><div class="kpi-value" style="color: #f59e0b;">{t1_cnt:,}</div><div class="kpi-subtitle">Audited CSPC EAS</div></div>', unsafe_allow_html=True)
+    with mk4:
+        lfm_cnt = (df_17k["catalog_intelligence_tier"] == "TIER_3_LASTFM_VERIFIED_ENTITY").sum()
+        st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #38bdf8;">Last.fm Verified</div><div class="kpi-value" style="color: #38bdf8;">{lfm_cnt:,}</div><div class="kpi-subtitle">Active Platform Entities</div></div>', unsafe_allow_html=True)
+    with mk5:
+        unver_cnt = len(df_17k) - ver_cnt
+        st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #ef4444;">Unaudited</div><div class="kpi-value" style="color: #ef4444;">{unver_cnt:,}</div><div class="kpi-subtitle">Zero-Registry Matches</div></div>', unsafe_allow_html=True)
+
+    st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
+
+    # 17k Table
+    render_cols = [
+        "catalog_rank", "tier_rank", "name", "catalog_intelligence_tier",
+        "audited_sales_units", "lastfm_listeners", "lastfm_playcount", "lastfm_url"
+    ]
+    disp_cols = [c for c in render_cols if c in filt_17k.columns]
+
+    st.dataframe(
+        filt_17k[disp_cols].reset_index(drop=True),
+        use_container_width=True,
+        height=560,
+        column_config={
+            "catalog_rank": st.column_config.NumberColumn("Global Rank", format="#%d", width="small"),
+            "tier_rank": st.column_config.NumberColumn("Tier Rank", format="#%d", width="small"),
+            "name": st.column_config.TextColumn("Artist Name", width="medium"),
+            "catalog_intelligence_tier": st.column_config.TextColumn("Registry Tier", width="medium"),
+            "audited_sales_units": st.column_config.NumberColumn("Audited Sales Units", format="%,d", width="medium"),
+            "lastfm_listeners": st.column_config.NumberColumn("Verified Listeners", format="%,d", width="medium"),
+            "lastfm_playcount": st.column_config.NumberColumn("Global Scrobbles", format="%,d", width="medium"),
+            "lastfm_url": st.column_config.LinkColumn("Last.fm Profile", width="small")
+        }
+    )
+
+    with st.sidebar:
+        st.markdown("##### 💾 17k Data Export")
+        csv_17k = filt_17k.to_csv(index=False).encode("utf-8")
+        st.download_button("📥 Export 17k Table CSV", data=csv_17k, file_name="master_17k_catalog.csv", mime="text/csv", use_container_width=True)
+
+# =====================================================================
+# MODULE 2: HIFFI HIP-HOP 500 (ORIGINAL EXECUTIVE DASHBOARD)
+# =====================================================================
+else:
+    df = df_500
+    with st.sidebar:
         view_mode = st.radio(
             "Leaderboard Perspective:",
             [
@@ -237,7 +317,6 @@ with st.sidebar:
             ],
             index=0,
         )
-
         st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
 
         catalog_filter = "All Monitored Artists (500)"
@@ -247,28 +326,24 @@ with st.sidebar:
                 "Filter Rows by Sales Telemetry:",
                 ["All Monitored Artists (500)", "Verified Only (321)", "Pure CSPC Only (62)"],
                 index=0,
-            help="Toggle between viewing the full cohort, verified certifications, or only ChartMasters studies.",
             )
             st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
 
         sim_weight = 0.15
         sim_mode = "matched_only"
-    if view_mode == "🧪 Commercial Simulation Engine (CPI)":
+        if view_mode == "🧪 Commercial Simulation Engine (CPI)" and engine is not None:
             st.markdown("##### 🧪 Simulation Parameters")
             sim_mode_choice = st.radio(
                 "Evaluation Scope:",
                 ["🎯 Matched Cohort Only", "🌐 Global Cohort (N=500, Weight Redistribution)"],
                 index=1 if catalog_filter == "All Monitored Artists (500)" else 0,
-            help="Matched Cohort evaluates cataloged artists in an isolated leaderboard. Global Cohort keeps all artists with normalized weights.",
             )
             sim_mode = "matched_only" if "Matched" in sim_mode_choice else "global_redistribution"
-
             sim_weight = st.select_slider(
                 "Commercial Signal Weight:",
                 options=[0.0, 0.05, 0.10, 0.15, 0.20],
                 value=0.15,
                 format_func=lambda x: f"{int(x*100)}% Commercial",
-            help="Blends Commercial Power Index (CPI) with baseline production score.",
             )
             st.caption(f"Baseline Score Weight: **{int((1.0 - sim_weight)*100)}%**")
             st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
@@ -279,7 +354,6 @@ with st.sidebar:
 
         tier_options = sorted(df["platform_evidence_tier"].dropna().unique().astype(int).tolist(), reverse=True)
         selected_tiers = st.multiselect("Evidence Tier", options=tier_options, default=tier_options)
-
         min_xp = st.slider("Min HIFFI-XP Score", 0.0, 100.0, 0.0, 5.0)
 
     # Filter logic
@@ -294,24 +368,30 @@ if catalog_filter == "Pure CSPC Only (62)":
     elif catalog_filter == "Verified Only (321)":
         filtered_df = filtered_df[filtered_df["sales_data_status"].isin(["VERIFIED_CSPC", "VERIFIED_RIAA"])].copy()
 
-# Execute simulation calculations if chosen
     sim_metrics = {}
-if view_mode == "🧪 Commercial Simulation Engine (CPI)":
+    if view_mode == "🧪 Commercial Simulation Engine (CPI)" and engine is not None:
+        try:
             filtered_df = engine.simulate_ranking(filtered_df, commercial_weight=sim_weight, mode=sim_mode)
             sim_metrics = engine.compute_simulation_metrics(filtered_df)
+        except Exception:
+            pass
 
     with st.sidebar:
         st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
         st.markdown("##### 💾 Unified Audit Exports")
 
+        if generate_executive_sales_workbook is not None:
+            try:
                 excel_workbook_bytes = generate_executive_sales_workbook(filtered_df, commercial_weight=sim_weight, mode=sim_mode)
                 st.download_button(
                     "📊 Multi-Tab Audit Excel (.xlsx)",
                     data=excel_workbook_bytes,
                     file_name="hiffi_executive_sales_audit.xlsx",
                     mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        width="stretch",
+                    use_container_width=True,
                 )
+            except Exception:
+                pass
 
         csv_payload = filtered_df.to_csv(index=False).encode("utf-8")
         st.download_button(
@@ -319,16 +399,14 @@ with st.sidebar:
             data=csv_payload,
             file_name="hiffi_multiplatform_filtered.csv",
             mime="text/csv",
-        width="stretch",
+            use_container_width=True,
         )
-
         st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
         st.caption("Governance: CSPC Methodology • EAS Units • Zero-Imputation Compliant")
 
-# ----------------- Main Header -----------------
     head_col1, head_col2 = st.columns([3, 1])
     with head_col1:
-    st.markdown("<h2 style='margin-bottom: 2px;'>HIFFI Multi-Platform Intelligence Hub</h2>", unsafe_allow_html=True)
+        st.markdown("<h2 style='margin-bottom: 2px;'>HIFFI Hip-Hop 500 Intelligence Hub</h2>", unsafe_allow_html=True)
         st.markdown("<p style='color: #9ca3af; font-size: 0.92rem; margin-top: 0;'>Multi-platform composite rankings with zero-imputation policy, dynamic weight redistribution, and verified record sales.</p>", unsafe_allow_html=True)
     with head_col2:
         st.markdown(
@@ -336,7 +414,7 @@ with head_col2:
             unsafe_allow_html=True,
         )
 
-# ----------------- Macro KPI Cards -----------------
+    # 500 Macro KPI Cards
     k1, k2, k3, k4, k5 = st.columns(5)
     with k1:
         st.markdown(f'<div class="kpi-box"><div class="kpi-title">Tracked Artists</div><div class="kpi-value">{len(df)}</div><div class="kpi-subtitle">Monitored Cohort</div></div>', unsafe_allow_html=True)
@@ -355,7 +433,6 @@ with k5:
 
     st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
 
-# ----------------- Dynamic Telemetry Coverage Banner -----------------
     if view_mode in ("💿 Record Sales (CSPC Methodology)", "🧪 Commercial Simulation Engine (CPI)"):
         verified_cspc_cnt = int((df["sales_data_status"] == "VERIFIED_CSPC").sum())
         verified_riaa_cnt = int((df["sales_data_status"] == "VERIFIED_RIAA").sum())
@@ -382,8 +459,6 @@ if view_mode in ("💿 Record Sales (CSPC Methodology)", "🧪 Commercial Simula
             unsafe_allow_html=True,
         )
 
-# ----------------- Standalone Module Views -----------------
-
     if view_mode == "💿 Record Sales (CSPC Methodology)":
         st.markdown("### 💿 Global Record Sales & Lifetime Commercial Ranking")
         st.caption("Standardized on ChartMasters CSPC, verified RIAA certifications, and video/audio streaming equivalent units.")
@@ -408,23 +483,15 @@ if view_mode == "💿 Record Sales (CSPC Methodology)":
         sales_table_df["provenance_badge"] = sales_table_df["sales_data_status"].apply(get_provenance_badge)
 
         sales_cols = [
-        "cohort_sales_rank",
+            "cohort_sales_rank", "chartmasters_sales_rank", "name", "provenance_badge",
-        "chartmasters_sales_rank",
+            "total_eas_cspc", "pure_album_sales", "streaming_eas", "physical_singles_sales",
-        "name",
+            "digital_singles_sales", "commercial_power_index", "hiffi_xp_published_rank"
-        "provenance_badge",
-        "total_eas_cspc",
-        "pure_album_sales",
-        "streaming_eas",
-        "physical_singles_sales",
-        "digital_singles_sales",
-        "commercial_power_index",
-        "hiffi_xp_published_rank",
         ]
         render_s_cols = [c for c in sales_cols if c in sales_table_df.columns]
 
         st.dataframe(
             sales_table_df[render_s_cols].reset_index(drop=True),
-        width="stretch",
+            use_container_width=True,
             height=380,
             column_config={
                 "cohort_sales_rank": st.column_config.NumberColumn("Cohort Sales Rank", format="#%d", width="small"),
@@ -442,7 +509,6 @@ if view_mode == "💿 Record Sales (CSPC Methodology)":
         )
 
         st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
-
         c_s1, c_s2 = st.columns(2)
         with c_s1:
             top_sellers = sales_table_df[sales_table_df["total_eas_cspc"] > 0].head(12)
@@ -451,40 +517,24 @@ if view_mode == "💿 Record Sales (CSPC Methodology)":
             fig_format.add_trace(go.Bar(name="Digital Singles", x=top_sellers["name"], y=top_sellers["digital_singles_sales"], marker_color="#38bdf8"))
             fig_format.add_trace(go.Bar(name="Streaming EAS", x=top_sellers["name"], y=top_sellers["streaming_eas"], marker_color="#10b981"))
             fig_format.update_layout(
-            barmode="stack",
+                barmode="stack", title="Format Contribution: Pure Albums vs Downloads vs Streaming EAS",
-            title="Format Contribution: Pure Albums vs Downloads vs Streaming EAS",
+                template="plotly_dark", paper_bgcolor="#111827", plot_bgcolor="#111827",
-            template="plotly_dark",
+                font=dict(color="#9ca3af"), height=360, margin=dict(l=20, r=20, t=50, b=20),
-            paper_bgcolor="#111827",
-            plot_bgcolor="#111827",
-            font=dict(color="#9ca3af"),
-            height=360,
-            margin=dict(l=20, r=20, t=50, b=20),
                 legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1.0),
             )
-        st.plotly_chart(fig_format, width="stretch")
+            st.plotly_chart(fig_format, use_container_width=True)
 
         with c_s2:
             scatter_data = sales_table_df.dropna(subset=["total_eas_cspc", "hiffi_xp_score"]).copy()
             fig_quad = px.scatter(
-            scatter_data,
+                scatter_data, x="hiffi_xp_score", y="total_eas_cspc", size="total_eas_cspc",
-            x="hiffi_xp_score",
+                color="provenance_badge", hover_name="name", log_y=True,
-            y="total_eas_cspc",
-            size="total_eas_cspc",
-            color="provenance_badge",
-            hover_name="name",
-            log_y=True,
                 title="Legacy vs. Velocity Quadrant (All-Time Sales vs HIFFI-XP)",
-            labels={"hiffi_xp_score": "Current Multi-Platform Velocity (HIFFI-XP)", "total_eas_cspc": "Total Lifetime EAS Units (Log Scale)"},
+                labels={"hiffi_xp_score": "Current Velocity (HIFFI-XP)", "total_eas_cspc": "Total Lifetime EAS Units (Log Scale)"},
                 template="plotly_dark",
             )
-        fig_quad.update_layout(
+            fig_quad.update_layout(paper_bgcolor="#111827", plot_bgcolor="#111827", font=dict(color="#9ca3af"), height=360, margin=dict(l=20, r=20, t=50, b=20))
-            paper_bgcolor="#111827",
+            st.plotly_chart(fig_quad, use_container_width=True)
-            plot_bgcolor="#111827",
-            font=dict(color="#9ca3af"),
-            height=360,
-            margin=dict(l=20, r=20, t=50, b=20),
-        )
-        st.plotly_chart(fig_quad, width="stretch")
 
     else:
         top3 = filtered_df.sort_values(by="hiffi_xp_published_rank", ascending=True).head(3)
@@ -566,15 +616,9 @@ else:
             comp_rank = "baseline_matched_rank" if sim_mode == "matched_only" else "hiffi_xp_published_rank"
 
             cols = [
-            "simulated_rank",
+                "simulated_rank", comp_rank, "rank_shift_indicator", "name",
-            comp_rank,
+                "simulated_hiffi_score", "hiffi_xp_score", "commercial_power_index",
-            "rank_shift_indicator",
+                "total_eas_cspc", "sales_data_status",
-            "name",
-            "simulated_hiffi_score",
-            "hiffi_xp_score",
-            "commercial_power_index",
-            "total_eas_cspc",
-            "sales_data_status",
             ]
         else:
             table_df = filtered_df.sort_values(by="hiffi_xp_published_rank", ascending=True).copy()
@@ -585,10 +629,9 @@ else:
             display_table["sales_data_status"] = display_table["sales_data_status"].apply(get_provenance_badge)
 
         render_cols = [c for c in cols if c in display_table.columns]
-
         st.dataframe(
             display_table[render_cols].reset_index(drop=True),
-        width="stretch",
+            use_container_width=True,
             height=340,
             column_config={
                 "hiffi_xp_published_rank": st.column_config.NumberColumn("Published Rank", format="%d", width="small"),
@@ -606,96 +649,29 @@ else:
         )
 
         st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
-
-    if view_mode == "🧪 Commercial Simulation Engine (CPI)":
-        c_sim1, c_sim2 = st.columns(2)
-        with c_sim1:
-            sim_delta_df = filtered_df.dropna(subset=["simulation_rank_delta"]).sort_values(by="total_eas_cspc", ascending=False).head(15).copy()
-            if not sim_delta_df.empty:
-                fig_delta = px.bar(
-                    sim_delta_df,
-                    x="name",
-                    y="simulation_rank_delta",
-                    color="simulation_rank_delta",
-                    color_continuous_scale="RdYlGn",
-                    title=f"Simulated Rank Movement at {int(sim_weight*100)}% Commercial Weight (Top 15 Sellers)",
-                    labels={"simulation_rank_delta": "Rank Movement (▲ Positive / ▼ Negative)", "name": "Artist"},
-                    template="plotly_dark",
-                )
-                fig_delta.update_layout(
-                    paper_bgcolor="#111827",
-                    plot_bgcolor="#111827",
-                    font=dict(color="#9ca3af"),
-                    margin=dict(l=20, r=20, t=50, b=20),
-                    height=350,
-                )
-                st.plotly_chart(fig_delta, width="stretch")
-
-        with c_sim2:
-            sensitivity_df = engine.run_sensitivity_matrix(filtered_df, mode=sim_mode).head(15)
-            fig_sens = go.Figure()
-            for idx, r in sensitivity_df.head(6).iterrows():
-                fig_sens.add_trace(go.Scatter(
-                    x=["0%", "5%", "10%", "15%", "20%"],
-                    y=[r["rank_0pct"], r["rank_5pct"], r["rank_10pct"], r["rank_15pct"], r["rank_20pct"]],
-                    mode="lines+markers",
-                    name=r["name"],
-                ))
-            fig_sens.update_layout(
-                title="Weight Sensitivity: Rank Trajectories Across Weights (Top Volatile Acts)",
-                yaxis=dict(autorange="reversed", title="Simulated Rank (Lower is Better)"),
-                xaxis=dict(title="Commercial Signal Weight (%)"),
-                template="plotly_dark",
-                paper_bgcolor="#111827",
-                plot_bgcolor="#111827",
-                font=dict(color="#9ca3af"),
-                margin=dict(l=20, r=20, t=50, b=20),
-                height=350,
-            )
-            st.plotly_chart(fig_sens, width="stretch")
-    else:
         c_left, c_right = st.columns(2)
         with c_left:
             chart_df = filtered_df.dropna(subset=["spotify_monthly_listeners", "instagram_followers", "hiffi_xp_score"]).copy()
             if not chart_df.empty:
                 fig_scatter = px.scatter(
-                    chart_df,
+                    chart_df, x="spotify_monthly_listeners", y="instagram_followers",
-                    x="spotify_monthly_listeners",
+                    size="hiffi_xp_score", color="platform_evidence_tier", hover_name="name",
-                    y="instagram_followers",
+                    log_x=True, log_y=True, title="Spotify Monthly Listeners vs. Instagram Followers",
-                    size="hiffi_xp_score",
-                    color="platform_evidence_tier",
-                    hover_name="name",
-                    log_x=True,
-                    log_y=True,
-                    title="Spotify Monthly Listeners vs. Instagram Followers",
                     labels={"spotify_monthly_listeners": "Spotify Monthly Listeners (Log Scale)", "instagram_followers": "Instagram Followers (Log Scale)", "platform_evidence_tier": "Tier"},
                     template="plotly_dark",
                 )
-                fig_scatter.update_layout(
+                fig_scatter.update_layout(paper_bgcolor="#111827", plot_bgcolor="#111827", font=dict(color="#9ca3af"), height=320, margin=dict(l=20, r=20, t=40, b=20))
-                    paper_bgcolor="#111827",
+                st.plotly_chart(fig_scatter, use_container_width=True)
-                    plot_bgcolor="#111827",
-                    font=dict(color="#9ca3af"),
-                    margin=dict(l=20, r=20, t=40, b=20),
-                    height=320,
-                )
-                st.plotly_chart(fig_scatter, width="stretch")
         with c_right:
             tier_counts = filtered_df["platform_evidence_tier"].value_counts().reset_index()
             tier_counts.columns = ["Tier", "Count"]
             fig_pie = px.pie(tier_counts, values="Count", names="Tier", title="Evidence Tier Cohort Distribution", template="plotly_dark", color="Tier", color_discrete_map={3: "#10b981", 2: "#38bdf8", 1: "#f59e0b", 0: "#ef4444"})
-            fig_pie.update_layout(
+            fig_pie.update_layout(paper_bgcolor="#111827", plot_bgcolor="#111827", font=dict(color="#9ca3af"), height=320, margin=dict(l=20, r=20, t=40, b=20))
-                paper_bgcolor="#111827",
+            st.plotly_chart(fig_pie, use_container_width=True)
-                plot_bgcolor="#111827",
-                font=dict(color="#9ca3af"),
-                margin=dict(l=20, r=20, t=40, b=20),
-                height=320,
-            )
-            st.plotly_chart(fig_pie, width="stretch")
 
-# ----------------- Deep Dive Scouting Dossier -----------------
+    # Dossier for 500
     st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
     st.markdown("#### 🔎 Artist Dossier")
-
     selected_name = st.selectbox("Inspect Artist Details:", df["name"].unique(), label_visibility="collapsed")
     row = df[df["name"] == selected_name].iloc[0]
 
@@ -728,8 +704,6 @@ st.markdown(
                 <div><b>Traditional Albums:</b> {f"{int(row.get('pure_album_sales')):,}" if pd.notna(row.get('pure_album_sales')) else "N/A"}</div>
                 <div><b>Streaming-Equivalent EAS:</b> {f"{int(row.get('streaming_eas')):,}" if pd.notna(row.get('streaming_eas')) else "N/A"}</div>
                 <div><b>Sales Status:</b> {get_provenance_badge(row.get('sales_data_status', ''))}</div>
-            <div><b>Effective Weights:</b> {row.get('effective_weights', 'N/A')}</div>
-            <div><b>Anomaly Flags:</b> {row.get('anomaly_flags', 'CLEAN')}</div>
             </div>
         </div>
         """,

scripts/apply_catalog_intelligence_tiers.py

@@ -0,0 +1,32 @@
+import os
+import pandas as pd
+
+DATA_DIR = "data"
+SALES_PATH = os.path.join(DATA_DIR, "master_sales_17k_verified.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+
+df = pd.read_csv(SALES_PATH)
+
+
+def assign_tier(row):
+  prov = str(row.get("provenance", ""))
+  units = row.get("audited_sales_units", 0)
+
+  if prov == "VERIFIED_CSPC":
+    return "TIER_1_GLOBAL_SUPERSTAR_CSPC"
+  elif prov == "VERIFIED_RIAA":
+    if pd.notna(units) and units >= 10_000_000:
+      return "TIER_2A_MAJOR_COMMERCIAL_CORE"
+    return "TIER_2B_CERTIFIED_CATALOG"
+
+  # Provenance UNVERIFIED_NO_RECORDS
+  return "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+
+
+df["catalog_intelligence_tier"] = df.apply(assign_tier, axis=1)
+
+print("--- CATALOG INTELLIGENCE TIER DISTRIBUTION ---")
+print(df["catalog_intelligence_tier"].value_counts())
+
+df.to_csv(OUTPUT_PATH, index=False)
+print(f"\nFinal catalog exported to: {OUTPUT_PATH}")

scripts/audit_sales_catalog.py

@@ -0,0 +1,20 @@
+import pandas as pd
+
+FILE_PATH = "data/master_sales_17k_verified.csv"
+df = pd.read_csv(FILE_PATH)
+
+print(f"Total rows in dataset: {len(df):,}")
+print("\n--- PROVENANCE BREAKDOWN ---")
+print(df["provenance"].value_counts())
+
+verified_df = df[df["provenance"] != "UNVERIFIED_NO_RECORDS"]
+print(f"\nTotal Verified Artists: {len(verified_df):,} ({(len(verified_df)/len(df))*100:.2f}%)")
+print(f"Total Unverified Artists: {len(df) - len(verified_df):,}")
+
+top_cols = ["sales_rank", "artist_name", "audited_sales_units", "provenance"]
+
+print("\n--- TOP 20 ARTISTS BY AUDITED SALES ---")
+print(verified_df[top_cols].sort_values("sales_rank").head(20).to_string(index=False))
+
+print("\n--- BOTTOM 5 VERIFIED ARTISTS (THRESHOLD CHECK) ---")
+print(verified_df[top_cols].sort_values("sales_rank", ascending=False).head(5).to_string(index=False))

scripts/build_verified_sales_catalog.py

@@ -0,0 +1,193 @@
+import os
+import re
+import sys
+import time
+import urllib.parse
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+import pandas as pd
+from rapidfuzz import fuzz
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import threading
+
+DATA_DIR = "data"
+CHECKPOINT_DIR = os.path.join(DATA_DIR, "checkpoints")
+os.makedirs(CHECKPOINT_DIR, exist_ok=True)
+
+MASTER_PATH = os.path.join(DATA_DIR, "master_artists_17k.csv")
+CSPC_PATH = os.path.join(DATA_DIR, "chartmasters_audited.csv")
+CHECKPOINT_FILE = os.path.join(CHECKPOINT_DIR, "sales_pipeline_progress.csv")
+FINAL_OUTPUT = os.path.join(DATA_DIR, "master_sales_17k_verified.csv")
+
+checkpoint_lock = threading.Lock()
+
+def normalize_text(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+def extract_units_from_title(title_text: str) -> int:
+    text = (title_text or "").upper().strip()
+    mult_match = re.search(r"(\d+)\s*X", text)
+    multiplier = int(mult_match.group(1)) if mult_match else 1
+    
+    if "DIAMOND" in text:
+        return max(multiplier, 1) * 10_000_000
+    elif "PLATINUM" in text or "MULTI-PLATINUM" in text:
+        return multiplier * 1_000_000
+    elif "GOLD" in text:
+        return 500_000
+    return 0
+
+def is_strict_match(target_name: str, candidate_artist: str, candidate_title: str) -> bool:
+    c_target = normalize_text(target_name)
+    if not c_target or len(c_target) < 2:
+        return False
+        
+    c_cand_artist = normalize_text(candidate_artist)
+    c_cand_title = normalize_text(candidate_title)
+    
+    if c_target == c_cand_artist:
+        return True
+    if re.search(rf"\b{re.escape(c_target)}\b", c_cand_artist):
+        return True
+    if re.search(rf"\b(feat|ft|featuring|with)\s+.*?\b{re.escape(c_target)}\b", candidate_title.lower()):
+        return True
+    if len(c_target) > 5 and fuzz.token_sort_ratio(c_target, c_cand_artist) >= 88:
+        return True
+        
+    return False
+
+def fetch_riaa(artist_name: str) -> dict:
+    target_clean = normalize_text(artist_name)
+    if not target_clean or len(target_clean) < 2:
+        return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+        
+    encoded = urllib.parse.quote_plus(target_clean)
+    url = f"https://www.riaa.com/gold-platinum/?tab_active=default-award&se={encoded}#search_section"
+    
+    try:
+        resp = requests.get(url, impersonate="chrome120", timeout=12)
+        if resp.status_code != 200:
+            return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+            
+        soup = BeautifulSoup(resp.text, "html.parser")
+        table = soup.find("table")
+        if not table:
+            return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+            
+        rows = table.find_all("tr")
+        total_units = 0
+        award_count = 0
+        
+        for r in rows[1:]:
+            cells = r.find_all("td")
+            if len(cells) < 3:
+                continue
+                
+            row_artist = cells[1].get_text(strip=True)
+            row_title = cells[2].get_text(strip=True)
+            
+            if not is_strict_match(artist_name, row_artist, row_title):
+                continue
+                    
+            img = cells[0].find("img")
+            units = extract_units_from_title(img.get("title", "") if img else "")
+            if units > 0:
+                total_units += units
+                award_count += 1
+                
+        if award_count > 0:
+            return {"units": total_units, "awards": award_count, "provenance": "VERIFIED_RIAA"}
+            
+    except Exception:
+        pass
+        
+    return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+
+def process_artist(row, cspc_dict):
+    a_id = row["artist_id"]
+    a_name = str(row["artist_name"])
+    c_name = normalize_text(a_name)
+    
+    # Tier 1: CSPC
+    if c_name in cspc_dict:
+        c_data = cspc_dict[c_name]
+        return {
+            "artist_id": a_id,
+            "artist_name": a_name,
+            "audited_sales_units": float(c_data["total_eas_cspc"]),
+            "awards_count": None,
+            "provenance": "VERIFIED_CSPC"
+        }
+    
+    # Tier 2: RIAA
+    riaa_res = fetch_riaa(a_name)
+    units = riaa_res["units"]
+    return {
+        "artist_id": a_id,
+        "artist_name": a_name,
+        "audited_sales_units": units if units > 0 else None,
+        "awards_count": riaa_res["awards"] if riaa_res["awards"] > 0 else None,
+        "provenance": riaa_res["provenance"]
+    }
+
+def run_pipeline(max_workers=5):
+    print(f"Initializing Multi-Threaded Sales Pipeline ({max_workers} Workers)...")
+    master_df = pd.read_csv(MASTER_PATH)
+    cspc_df = pd.read_csv(CSPC_PATH)
+    cspc_dict = {normalize_text(r["artist_name"]): r for _, r in cspc_df.iterrows()}
+    
+    if os.path.exists(CHECKPOINT_FILE):
+        progress_df = pd.read_csv(CHECKPOINT_FILE)
+        processed_ids = set(progress_df["artist_id"].dropna())
+        records = progress_df.to_dict("records")
+        print(f"Resuming from checkpoint: {len(processed_ids):,} artists already processed.")
+    else:
+        records = []
+        processed_ids = set()
+
+    queue_df = master_df[~master_df["artist_id"].isin(processed_ids)]
+    total_remaining = len(queue_df)
+    total_catalog = len(master_df)
+    
+    print(f"Running on remaining {total_remaining:,} artists out of {total_catalog:,}...\n")
+    
+    processed_count = len(processed_ids)
+    
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_to_artist = {
+            executor.submit(process_artist, row, cspc_dict): row 
+            for _, row in queue_df.iterrows()
+        }
+        
+        for future in as_completed(future_to_artist):
+            res = future.result()
+            records.append(res)
+            processed_count += 1
+            
+            if res["audited_sales_units"] is not None and res["audited_sales_units"] > 0:
+                print(f"[{processed_count:,}/{total_catalog:,}] {res['artist_name']:<22} -> {res['audited_sales_units']:>12,f} ({res['provenance']})")
+            
+            # Save checkpoint every 50 artists
+            if processed_count % 50 == 0:
+                with checkpoint_lock:
+                    pd.DataFrame(records).to_csv(CHECKPOINT_FILE, index=False)
+
+    # Final Rank & Export
+    df_out = pd.DataFrame(records)
+    has_sales = df_out["audited_sales_units"].notna()
+    df_out["sales_rank"] = None
+    df_out.loc[has_sales, "sales_rank"] = (
+        df_out.loc[has_sales, "audited_sales_units"]
+        .rank(ascending=False, method="min")
+        .astype(int)
+    )
+    
+    df_out.to_csv(FINAL_OUTPUT, index=False)
+    print(f"\nPipeline successfully finished! Final catalog saved to: {FINAL_OUTPUT}")
+
+if __name__ == "__main__":
+    run_pipeline(max_workers=5)

scripts/enrich_with_kworb_listeners.py

@@ -0,0 +1,144 @@
+import os
+import re
+import time
+import pandas as pd
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+
+DATA_DIR = "data"
+SALES_PATH = os.path.join(DATA_DIR, "master_sales_17k_verified.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+
+KWORB_STREAMS_URL = "https://kworb.net/spotify/artists.html"
+KWORB_LISTENERS_URL = "https://kworb.net/spotify/listeners.html"
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def fetch_with_retry(session: requests.Session, url: str, max_retries: int = 3, timeout: int = 45):
+    for attempt in range(1, max_retries + 1):
+        try:
+            resp = session.get(url, impersonate="chrome120", timeout=timeout)
+            if resp.status_code == 200:
+                return resp
+            print(f"Warning: HTTP {resp.status_code} on attempt {attempt}/{max_retries}. Retrying...")
+        except Exception as e:
+            print(f"Connection attempt {attempt}/{max_retries} timed out or failed ({e}). Retrying in 3s...")
+            time.sleep(3)
+    raise ConnectionError(f"Failed to fetch {url} after {max_retries} attempts.")
+
+
+def fetch_kworb_streams(session: requests.Session) -> dict:
+    print("1/2: Fetching Kworb all-time streams table...")
+    resp = fetch_with_retry(session, KWORB_STREAMS_URL, max_retries=3, timeout=45)
+
+    soup = BeautifulSoup(resp.text, "html.parser")
+    table = soup.find("table")
+    streams_data = {}
+
+    for row in table.find_all("tr")[1:]:
+        cells = row.find_all("td")
+        if len(cells) < 2:
+            continue
+        artist_anchor = cells[0].find("a")
+        artist_name = artist_anchor.get_text(strip=True) if artist_anchor else cells[0].get_text(strip=True)
+
+        try:
+            stream_str = cells[1].get_text(strip=True).replace(",", "")
+            total_streams = int(float(stream_str) * 1_000_000)
+        except (ValueError, IndexError):
+            total_streams = 0
+
+        norm_key = normalize_name(artist_name)
+        if norm_key and total_streams > 0:
+            streams_data[norm_key] = total_streams
+
+    print(f"Loaded {len(streams_data):,} stream records from Kworb.")
+    return streams_data
+
+
+def fetch_kworb_listeners(session: requests.Session) -> dict:
+    print("2/2: Fetching Kworb monthly listeners table...")
+    resp = fetch_with_retry(session, KWORB_LISTENERS_URL, max_retries=3, timeout=45)
+
+    soup = BeautifulSoup(resp.text, "html.parser")
+    table = soup.find("table")
+    listeners_data = {}
+
+    for row in table.find_all("tr")[1:]:
+        cells = row.find_all("td")
+        if len(cells) < 3:
+            continue
+        artist_anchor = cells[1].find("a")
+        artist_name = artist_anchor.get_text(strip=True) if artist_anchor else cells[1].get_text(strip=True)
+
+        try:
+            listeners_str = cells[2].get_text(strip=True).replace(",", "")
+            monthly_listeners = int(listeners_str)
+        except (ValueError, IndexError):
+            monthly_listeners = 0
+
+        norm_key = normalize_name(artist_name)
+        if norm_key and monthly_listeners > 0:
+            listeners_data[norm_key] = monthly_listeners
+
+    print(f"Loaded {len(listeners_data):,} monthly listener records from Kworb.")
+    return listeners_data
+
+
+def run_strict_intelligence_enrichment():
+    print("Loading base sales dataset...")
+    df = pd.read_csv(SALES_PATH)
+
+    session = requests.Session()
+    streams_dict = fetch_kworb_streams(session)
+    listeners_dict = fetch_kworb_listeners(session)
+
+    print("Mapping metrics to normalized artist keys...")
+    df["norm_key"] = df["artist_name"].apply(normalize_name)
+
+    # Attach verified stream and listener metrics via exact match
+    df["spotify_total_streams"] = df["norm_key"].map(streams_dict)
+    df["spotify_monthly_listeners"] = df["norm_key"].map(listeners_dict)
+
+    def assign_tier(row):
+        prov = str(row.get("provenance", ""))
+        units = row.get("audited_sales_units", 0)
+
+        # Baseline Audits
+        if prov == "VERIFIED_CSPC":
+            return "TIER_1_GLOBAL_SUPERSTAR_CSPC"
+        elif prov == "VERIFIED_RIAA":
+            if pd.notna(units) and units >= 10_000_000:
+                return "TIER_2A_MAJOR_COMMERCIAL_CORE"
+            return "TIER_2B_CERTIFIED_CATALOG"
+
+        # Streaming Velocity (Exact-match verified only)
+        if pd.notna(row["spotify_total_streams"]) and row["spotify_total_streams"] > 0:
+            return "TIER_3A_VERIFIED_DIGITAL_CATALOG"
+
+        if pd.notna(row["spotify_monthly_listeners"]) and row["spotify_monthly_listeners"] > 0:
+            return "TIER_3B_ACTIVE_MONTHLY_STREAMERS"
+
+        return "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+
+    print("Assigning finalized intelligence tiers...")
+    df["catalog_intelligence_tier"] = df.apply(assign_tier, axis=1)
+
+    df.drop(columns=["norm_key"], inplace=True)
+
+    print("\n--- FINAL HARDENED TIER BREAKDOWN ---")
+    print(df["catalog_intelligence_tier"].value_counts())
+
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nSaved sanitized catalog to: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_strict_intelligence_enrichment()

scripts/enrich_with_kworb_streams.py

@@ -0,0 +1,128 @@
+import os
+import re
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+import pandas as pd
+from rapidfuzz import fuzz
+
+DATA_DIR = "data"
+CATALOG_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+
+KWORB_STREAMS_URL = "https://kworb.net/spotify/artists.html"
+KWORB_LISTENERS_URL = "https://kworb.net/spotify/listeners.html"
+
+
+def normalize_name(text: str) -> str:
+  if not text:
+    return ""
+  cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+  cleaned = re.sub(r"[^\w\s]", "", cleaned)
+  return cleaned.strip().lower()
+
+
+def fetch_kworb_master_table() -> dict:
+  print("Fetching Kworb master stream table...")
+  session = requests.Session()
+  resp = session.get(KWORB_STREAMS_URL, impersonate="chrome120", timeout=15)
+
+  if resp.status_code != 200:
+    raise ConnectionError(
+        f"Failed to fetch Kworb streams table (Status: {resp.status_code})"
+    )
+
+  soup = BeautifulSoup(resp.text, "html.parser")
+  table = soup.find("table")
+  if not table:
+    raise ValueError("No table found on Kworb page.")
+
+  kworb_data = {}
+  rows = table.find_all("tr")
+
+  # Table format: Artist | Total Streams (M) | Daily (M) | As Lead (M) | Solo (M) | As Feature (M)
+  for row in rows[1:]:
+    cells = row.find_all("td")
+    if len(cells) < 2:
+      continue
+
+    artist_anchor = cells[0].find("a")
+    artist_name = (
+        artist_anchor.get_text(strip=True)
+        if artist_anchor
+        else cells[0].get_text(strip=True)
+    )
+
+    try:
+      # Kworb streams are formatted in millions (e.g., '143,818.8' means 143.8 billion)
+      stream_str = cells[1].get_text(strip=True).replace(",", "")
+      total_streams = int(float(stream_str) * 1_000_000)
+    except (ValueError, IndexError):
+      total_streams = 0
+
+    norm_key = normalize_name(artist_name)
+    if norm_key and total_streams > 0:
+      kworb_data[norm_key] = {
+          "kworb_artist_name": artist_name,
+          "spotify_total_streams": total_streams,
+      }
+
+  print(f"Loaded {len(kworb_data):,} artists with lifetime stream metrics.")
+  return kworb_data
+
+
+def run_kworb_enrichment():
+  df = pd.read_csv(CATALOG_PATH)
+  kworb_dict = fetch_kworb_master_table()
+
+  if "spotify_total_streams" not in df.columns:
+    df["spotify_total_streams"] = None
+
+  unverified_mask = df["catalog_intelligence_tier"].str.contains(
+      "TIER_3_UNAUDITED", na=False
+  )
+  unverified_indices = df[unverified_mask].index
+
+  print(
+      f"Matching Kworb streaming metrics against {len(unverified_indices):,}"
+      " unverified artists..."
+  )
+
+  matches = 0
+  for idx in unverified_indices:
+    a_name = str(df.at[idx, "artist_name"])
+    c_name = normalize_name(a_name)
+
+    # 1. Exact match
+    if c_name in kworb_dict:
+      entry = kworb_dict[c_name]
+      df.at[idx, "spotify_total_streams"] = entry["spotify_total_streams"]
+      df.at[idx, "catalog_intelligence_tier"] = (
+          "TIER_3A_VERIFIED_DIGITAL_CATALOG"
+      )
+      matches += 1
+      continue
+
+    # 2. Strict token match for names longer than 5 chars
+    if len(c_name) > 5:
+      for k_key, entry in kworb_dict.items():
+        if fuzz.token_sort_ratio(c_name, k_key) >= 92:
+          df.at[idx, "spotify_total_streams"] = entry["spotify_total_streams"]
+          df.at[idx, "catalog_intelligence_tier"] = (
+              "TIER_3A_VERIFIED_DIGITAL_CATALOG"
+          )
+          matches += 1
+          break
+
+  print(
+      f"\nEnrichment complete! Successfully matched {matches:,} previously"
+      " unverified artists."
+  )
+  print("\nUpdated Catalog Intelligence Tiers:")
+  print(df["catalog_intelligence_tier"].value_counts())
+
+  df.to_csv(OUTPUT_PATH, index=False)
+  print(f"Saved updated catalog to: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+  run_kworb_enrichment()

scripts/filter_recovery_collisions.py

@@ -0,0 +1,46 @@
+import difflib
+import urllib.parse
+import pandas as pd
+
+DATA_PATH = "data/master_artists_intelligence_17k_final.csv"
+
+df = pd.read_csv(DATA_PATH)
+
+recovered_mask = df["catalog_intelligence_tier"] == "TIER_3_LASTFM_VERIFIED_ENTITY"
+
+reverted_count = 0
+for idx in df[recovered_mask].index:
+    orig = str(df.at[idx, "artist_name"]).strip()
+    url = str(df.at[idx, "lastfm_url"]).strip()
+
+    if "/music/" not in url:
+        continue
+
+    # Extract slug after /music/
+    slug = url.split("/music/")[-1]
+    # Remove Last.fm internal redirect markers
+    slug = slug.replace("+noredirect/", "").replace("noredirect/", "")
+    # Unquote URL encodings (%20 -> space, %21 -> !, etc.)
+    matched_name = urllib.parse.unquote_plus(slug).strip()
+
+    # Clean characters for comparison
+    clean_orig = "".join(c for c in orig.lower() if c.isalnum() or c.isspace()).strip()
+    clean_match = "".join(c for c in matched_name.lower() if c.isalnum() or c.isspace()).strip()
+
+    # Direct exception for known collisions
+    if clean_orig == "tkkravitz" and "lenny" in clean_match:
+        similarity = 0.0
+    else:
+        similarity = difflib.SequenceMatcher(None, clean_orig, clean_match).ratio()
+
+    # If the name drifted completely, revert to unaudited
+    if similarity < 0.65:
+        reverted_count += 1
+        print(f"Reverting false collision: '{orig}' -> matched '{matched_name}' (Sim: {similarity:.2f})")
+        df.at[idx, "lastfm_listeners"] = None
+        df.at[idx, "lastfm_playcount"] = None
+        df.at[idx, "lastfm_url"] = None
+        df.at[idx, "catalog_intelligence_tier"] = "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+
+print(f"\nFiltered out {reverted_count} true false collisions.")
+df.to_csv(DATA_PATH, index=False)

scripts/finalize_master_catalog.py

@@ -0,0 +1,72 @@
+import os
+import re
+import pandas as pd
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+FINAL_OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def run_final_qa():
+    print(f"Reading dataset: {INPUT_PATH}")
+    df = pd.read_csv(INPUT_PATH)
+    initial_count = len(df)
+    print(f"Initial total artists: {initial_count:,}")
+
+    # 1. Deduplication by normalized name and artist_id
+    df["norm_key"] = df["artist_name"].apply(normalize_name)
+    
+    # Sort so verified records take priority over unverified during deduplication
+    tier_order = {
+        "TIER_1_GLOBAL_SUPERSTAR_CSPC": 1,
+        "TIER_2A_MAJOR_COMMERCIAL_CORE": 2,
+        "TIER_2B_CERTIFIED_CATALOG": 3,
+        "TIER_3A_VERIFIED_DIGITAL_CATALOG": 4,
+        "TIER_3B_ACTIVE_MONTHLY_STREAMERS": 5,
+        "TIER_3_UNAUDITED_DIGITAL_CATALOG": 6,
+    }
+    df["tier_rank"] = df["catalog_intelligence_tier"].map(tier_order).fillna(99)
+    df.sort_values(by=["tier_rank", "audited_sales_units", "spotify_total_streams", "spotify_monthly_listeners"], 
+                   ascending=[True, False, False, False], inplace=True)
+
+    # Drop duplicate normalized names, retaining highest-ranked tier
+    df.drop_duplicates(subset=["norm_key"], keep="first", inplace=True)
+    deduped_count = len(df)
+    print(f"Deduplicated count: {deduped_count:,} (Removed {initial_count - deduped_count:,} duplicate variants)")
+
+    # 2. Add Unified Sorting Metric Columns for Downstream UI
+    def get_primary_metric(row):
+        tier = row["catalog_intelligence_tier"]
+        if tier in ["TIER_1_GLOBAL_SUPERSTAR_CSPC", "TIER_2A_MAJOR_COMMERCIAL_CORE", "TIER_2B_CERTIFIED_CATALOG"]:
+            return pd.Series(["AUDITED_SALES_UNITS", row["audited_sales_units"]])
+        elif tier == "TIER_3A_VERIFIED_DIGITAL_CATALOG":
+            return pd.Series(["LIFETIME_STREAMS", row["spotify_total_streams"]])
+        elif tier == "TIER_3B_ACTIVE_MONTHLY_STREAMERS":
+            return pd.Series(["MONTHLY_LISTENERS", row["spotify_monthly_listeners"]])
+        return pd.Series(["UNAUDITED", None])
+
+    df[["primary_metric_type", "primary_metric_value"]] = df.apply(get_primary_metric, axis=1)
+
+    # Clean temporary helper columns
+    df.drop(columns=["norm_key", "tier_rank"], inplace=True)
+
+    # Sort final export: Tiered priority first, then primary metric descending
+    df.sort_values(by=["catalog_intelligence_tier", "primary_metric_value"], 
+                   ascending=[True, False], inplace=True)
+
+    df.to_csv(FINAL_OUTPUT_PATH, index=False)
+    print(f"\nFinal hardened intelligence catalog exported to: {FINAL_OUTPUT_PATH}")
+    print("\nFinal Tier Breakdown:")
+    print(df["catalog_intelligence_tier"].value_counts())
+
+
+if __name__ == "__main__":
+    run_final_qa()

scripts/rank_master_catalog.py

@@ -0,0 +1,97 @@
+import os
+import pandas as pd
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+
+
+def run_empirical_catalog_ranking():
+    if not os.path.exists(INPUT_PATH):
+        print(f"ERROR: Dataset not found at {INPUT_PATH}")
+        return
+
+    print(f"Loading master verified catalog: {INPUT_PATH}")
+    df = pd.read_csv(INPUT_PATH)
+
+    # 1. Define Strict Registry Tier Hierarchy
+    tier_priority = {
+        "TIER_1_GLOBAL_SUPERSTAR_CSPC": 1,
+        "TIER_2A_MAJOR_COMMERCIAL_CORE": 2,
+        "TIER_2B_CERTIFIED_CATALOG": 3,
+        "TIER_3A_VERIFIED_DIGITAL_CATALOG": 4,
+        "TIER_3B_ACTIVE_MONTHLY_STREAMERS": 5,
+        "TIER_3_LASTFM_VERIFIED_ENTITY": 6,
+        "TIER_3_UNAUDITED_DIGITAL_CATALOG": 7
+    }
+
+    df["tier_order"] = df["catalog_intelligence_tier"].map(tier_priority).fillna(99)
+
+    # 2. Extract Primary Audited Metric per Tier (No conversions, pure integers)
+    def extract_tier_metric(row):
+        order = row["tier_order"]
+        if order in [1, 2, 3]:
+            # CSPC EAS / RIAA Certified Units
+            val = row.get("audited_sales_units")
+            return float(val) if pd.notna(val) else 0.0
+        elif order == 4:
+            # Kworb Spotify Total Discography Streams
+            val = row.get("spotify_total_streams")
+            return float(val) if pd.notna(val) else 0.0
+        elif order == 5:
+            # Kworb Spotify Peak Monthly Listeners
+            val = row.get("spotify_monthly_listeners")
+            return float(val) if pd.notna(val) else 0.0
+        elif order == 6:
+            # Last.fm Global Verified Listeners
+            val = row.get("lastfm_listeners")
+            return float(val) if pd.notna(val) else 0.0
+        return 0.0
+
+    df["primary_empirical_metric"] = df.apply(extract_tier_metric, axis=1)
+
+    # 3. Sort strictly by Tier Order (1 -> 6), then by Primary Metric (Highest -> Lowest)
+    df.sort_values(
+        by=["tier_order", "primary_empirical_metric"],
+        ascending=[True, False],
+        inplace=True
+    )
+    df.reset_index(drop=True, inplace=True)
+
+    # 4. Generate Global Catalog Rank (Only for verified tiers 1 through 6)
+    verified_mask = df["tier_order"] <= 6
+    df["catalog_rank"] = None
+    df.loc[verified_mask, "catalog_rank"] = range(1, int(verified_mask.sum()) + 1)
+
+    # 5. Generate Peer Tier Rank
+    df["tier_rank"] = None
+    for tier_name in df["catalog_intelligence_tier"].unique():
+        if tier_name != "TIER_3_UNAUDITED_DIGITAL_CATALOG":
+            t_mask = df["catalog_intelligence_tier"] == tier_name
+            df.loc[t_mask, "tier_rank"] = range(1, int(t_mask.sum()) + 1)
+
+    # Clean up temporary helper columns
+    df.drop(columns=["tier_order", "primary_empirical_metric"], inplace=True)
+
+    # Save finalized ranked dataset
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nEmpirical ranking complete! Saved to {OUTPUT_PATH}")
+
+    # Display Top 15 Overall
+    preview_cols = [
+        col for col in ["catalog_rank", "tier_rank", "artist_name", "catalog_intelligence_tier", "audited_sales_units", "lastfm_listeners"]
+        if col in df.columns
+    ]
+    print("\n--- TOP 15 CATALOG ARTISTS ---")
+    print(df[preview_cols].head(15).to_string(index=False))
+
+    # Display Top 5 in Tier 3 Last.fm Verified Entities
+    lastfm_sample = df[df["catalog_intelligence_tier"] == "TIER_3_LASTFM_VERIFIED_ENTITY"]
+    if not lastfm_sample.empty:
+        print("\n--- TOP 5 TIER 3 (LAST.FM VERIFIED) ARTISTS ---")
+        lfm_cols = ["catalog_rank", "tier_rank", "artist_name", "lastfm_listeners", "lastfm_playcount"]
+        print(lastfm_sample[lfm_cols].head(5).to_string(index=False))
+
+
+if __name__ == "__main__":
+    run_empirical_catalog_ranking()

scripts/recover_367_unverified.py

@@ -0,0 +1,105 @@
+import os
+import re
+import time
+import ftfy
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATA_PATH = "data/master_artists_intelligence_17k_final.csv"
+OUTPUT_PATH = "data/master_artists_intelligence_17k_final.csv"
+LASTFM_API_KEY = os.getenv("LASTFM_API_KEY")
+
+
+def clean_artist_query(raw_name: str) -> str:
+    """Fix mojibake encoding and clean handle artifacts."""
+    if not raw_name or pd.isna(raw_name):
+        return ""
+    
+    # 1. Fix Mojibake encoding (ãl -> al, Amoré -> Amoré)
+    name = ftfy.fix_text(str(raw_name))
+
+    # 2. Strip leading numbers attached to handles (e.g. 1dessdior -> dess dior)
+    name = re.sub(r"^[0-9]+([a-zA-Z])", r"\1", name)
+
+    # 3. Strip trailing underscores or dots commonly found in IG handles
+    name = re.sub(r"[_\.]+$", "", name)
+    name = name.replace("_", " ").replace(".", " ")
+
+    # 4. Remove collaboration trails like & the Brand New S
+    if " & the " in name:
+        name = name.split(" & the ")[0]
+
+    return name.strip()
+
+
+def query_lastfm_search(session: requests.Session, query: str) -> dict:
+    url = "https://ws.audioscrobbler.com/2.0/"
+    params = {
+        "method": "artist.search",
+        "artist": query,
+        "api_key": LASTFM_API_KEY,
+        "format": "json",
+        "limit": 3
+    }
+    headers = {"User-Agent": "HIFFIRecovery/1.0 (srilekhar86@gmail.com)"}
+
+    try:
+        resp = session.get(url, params=params, headers=headers, impersonate="chrome120", timeout=8)
+        if resp.status_code == 200:
+            data = resp.json()
+            matches = data.get("results", {}).get("artistmatches", {}).get("artist", [])
+            if isinstance(matches, dict):
+                matches = [matches]
+            if matches:
+                # Pick the top result with registered listeners
+                for match in matches:
+                    listeners = int(match.get("listeners", 0))
+                    if listeners > 0:
+                        return {
+                            "canonical_name": match.get("name"),
+                            "listeners": listeners,
+                            "url": match.get("url")
+                        }
+    except Exception:
+        pass
+    return None
+
+
+def run_recovery():
+    print(f"Loading master catalog: {DATA_PATH}")
+    df = pd.read_csv(DATA_PATH)
+
+    unv_mask = df["catalog_rank"].isna() | (df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG")
+    unv_indices = df[unv_mask].index.tolist()
+    print(f"Targeting {len(unv_indices)} unverified candidates for registry recovery...\n")
+
+    session = requests.Session()
+    recovered = 0
+
+    for idx in unv_indices:
+        raw_name = df.at[idx, "artist_name"]
+        cleaned_query = clean_artist_query(raw_name)
+
+        if not cleaned_query or len(cleaned_query) <= 1:
+            continue
+
+        res = query_lastfm_search(session, cleaned_query)
+        time.sleep(0.2)  # Respect Last.fm rate limits
+
+        if res:
+            df.at[idx, "lastfm_listeners"] = res["listeners"]
+            df.at[idx, "lastfm_url"] = res["url"]
+            df.at[idx, "catalog_intelligence_tier"] = "TIER_3_LASTFM_VERIFIED_ENTITY"
+            recovered += 1
+            print(f"[{recovered}] Recovered: '{raw_name}' -> '{res['canonical_name']}' ({res['listeners']:,} listeners)")
+
+    print(f"\nRecovery pass complete: {recovered} / {len(unv_indices)} artists recovered.")
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"Master file updated: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_recovery()

scripts/recover_unverified_artists.py

@@ -0,0 +1,127 @@
+import os
+import re
+import time
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATA_PATH = "data/master_artists_intelligence_17k_final.csv"
+OUTPUT_PATH = "data/master_artists_intelligence_17k_final.csv"
+LASTFM_API_KEY = os.getenv("LASTFM_API_KEY")
+
+
+def clean_primary_artist(raw_name: str) -> str:
+    """Extracts primary artist if string is a collaboration or feature."""
+    if not raw_name or pd.isna(raw_name):
+        return ""
+    text = str(raw_name)
+    # Strip (feat. ...), [prod. ...], etc.
+    text = re.sub(r"\(.*?\)|\[.*?\]", "", text)
+    # Split on explicit collaboration delimiters and take primary artist
+    for delim in [" feat. ", " ft. ", " featuring ", " Feat. ", " Ft. ", " & ", " x ", " X ", " vs. ", " vs "]:
+        if delim in text:
+            text = text.split(delim)[0]
+    return text.strip()
+
+
+def query_lastfm_search(session: requests.Session, artist_name: str) -> dict:
+    url = "https://ws.audioscrobbler.com/2.0/"
+    params = {
+        "method": "artist.search",
+        "artist": artist_name,
+        "api_key": LASTFM_API_KEY,
+        "format": "json",
+        "limit": 1
+    }
+    headers = {"User-Agent": "HIFFIRecovery/1.0 (annotation783@gmail.com)"}
+
+    try:
+        resp = session.get(url, params=params, headers=headers, impersonate="chrome120", timeout=10)
+        if resp.status_code == 200:
+            data = resp.json()
+            matches = data.get("results", {}).get("artistmatches", {}).get("artist", [])
+            if matches:
+                top_match = matches[0] if isinstance(matches, list) else matches
+                listeners = int(top_match.get("listeners", 0))
+                if listeners > 0:
+                    return {
+                        "verified_name": top_match.get("name"),
+                        "lastfm_listeners": listeners,
+                        "lastfm_url": top_match.get("url"),
+                        "source": "LASTFM_SEARCH"
+                    }
+    except Exception:
+        pass
+    return None
+
+
+def query_musicbrainz(session: requests.Session, artist_name: str) -> dict:
+    """Open authoritative music registry fallback."""
+    url = "https://musicbrainz.org/ws/2/artist/"
+    params = {"query": f'artist:"{artist_name}"', "fmt": "json", "limit": 1}
+    headers = {"User-Agent": "HIFFIArtistIntelligence/1.0 ( srilekhar86@gmail.com )"}
+
+    try:
+        resp = session.get(url, params=params, headers=headers, timeout=10)
+        if resp.status_code == 200:
+            data = resp.json()
+            artists = data.get("artists", [])
+            if artists:
+                top_artist = artists[0]
+                score = int(top_artist.get("score", 0))
+                if score >= 90:  # High confidence registry match
+                    mbid = top_artist.get("id")
+                    return {
+                        "verified_name": top_artist.get("name"),
+                        "lastfm_listeners": None,
+                        "lastfm_url": f"https://musicbrainz.org/artist/{mbid}",
+                        "source": "MUSICBRAINZ_REGISTRY"
+                    }
+    except Exception:
+        pass
+    return None
+
+
+def run_recovery():
+    df = pd.read_csv(DATA_PATH)
+    unverified_mask = df["catalog_rank"].isna() | (df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG")
+    indices_to_recover = df[unverified_mask].index.tolist()
+
+    print(f"Attempting secondary recovery on {len(indices_to_recover):,} unverified artists...\n")
+
+    session = requests.Session()
+    recovered_count = 0
+
+    for idx in indices_to_recover:
+        raw_name = str(df.at[idx, "artist_name"])
+        cleaned = clean_primary_artist(raw_name)
+
+        # 1. Try Last.fm search endpoint with cleaned name
+        res = query_lastfm_search(session, cleaned if cleaned else raw_name)
+        time.sleep(0.3)
+
+        # 2. If no result, try MusicBrainz open registry
+        if not res:
+            res = query_musicbrainz(session, cleaned if cleaned else raw_name)
+            time.sleep(1.0)  # MusicBrainz rate limit compliance
+
+        if res:
+            recovered_count += 1
+            if res["lastfm_listeners"]:
+                df.at[idx, "lastfm_listeners"] = res["lastfm_listeners"]
+            if res["lastfm_url"]:
+                df.at[idx, "lastfm_url"] = res["lastfm_url"]
+
+            tier_label = "TIER_3_LASTFM_VERIFIED_ENTITY" if res["source"] == "LASTFM_SEARCH" else "TIER_3_MUSICBRAINZ_VERIFIED"
+            df.at[idx, "catalog_intelligence_tier"] = tier_label
+            print(f"✅ Recovered: '{raw_name}' -> '{res['verified_name']}' ({tier_label})")
+
+    print(f"\nRecovery complete: Successfully recovered {recovered_count:,} additional entities.")
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"Updated catalog saved to {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_recovery()

scripts/verify_artists_lastfm.py

@@ -0,0 +1,180 @@
+import os
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+CHECKPOINT_PATH = os.path.join(DATA_DIR, "checkpoints", "lastfm_verification_checkpoint.csv")
+
+os.makedirs(os.path.join(DATA_DIR, "checkpoints"), exist_ok=True)
+
+LASTFM_API_KEY = os.getenv("LASTFM_API_KEY")
+NUM_WORKERS = 2  # Safe concurrency limit
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def fetch_artist_lastfm(session: requests.Session, a_name: str) -> dict:
+    target_norm = normalize_name(a_name)
+    url = "https://ws.audioscrobbler.com/2.0/"
+    params = {
+        "method": "artist.getinfo",
+        "artist": a_name,
+        "api_key": LASTFM_API_KEY,
+        "format": "json",
+        "autocorrect": "0"
+    }
+    headers = {"User-Agent": "HIFFIArtistAudit/1.0 (annotation783@gmail.com)"}
+
+    for attempt in range(4):
+        try:
+            resp = session.get(url, params=params, headers=headers, impersonate="chrome120", timeout=12)
+            if resp.status_code == 200:
+                data = resp.json()
+
+                # Handle Last.fm Error 29: Rate Limit Exceeded
+                if data.get("error") == 29:
+                    time.sleep(15)
+                    continue
+
+                artist_data = data.get("artist")
+                if artist_data and "name" in artist_data:
+                    ret_norm = normalize_name(artist_data.get("name", ""))
+                    if ret_norm == target_norm:
+                        stats = artist_data.get("stats", {})
+                        listeners = int(stats.get("listeners", 0))
+                        playcount = int(stats.get("playcount", 0))
+                        url_link = artist_data.get("url", "")
+
+                        return {
+                            "artist_name": a_name,
+                            "lastfm_listeners": listeners,
+                            "lastfm_playcount": playcount,
+                            "lastfm_url": url_link,
+                            "verified": listeners > 0
+                        }
+                break
+
+            elif resp.status_code in [429, 503]:
+                time.sleep(12)
+                continue
+
+        except Exception:
+            time.sleep(3)
+
+    return {
+        "artist_name": a_name,
+        "lastfm_listeners": None,
+        "lastfm_playcount": None,
+        "lastfm_url": None,
+        "verified": False
+    }
+
+
+def run_lastfm_verification():
+    if not LASTFM_API_KEY:
+        print("ERROR: LASTFM_API_KEY not found in .env.")
+        return
+
+    df = pd.read_csv(INPUT_PATH)
+
+    for col in ["lastfm_listeners", "lastfm_playcount", "lastfm_url"]:
+        if col not in df.columns:
+            df[col] = None
+
+    # Load verified artists from checkpoint
+    verified_map = {}
+    if os.path.exists(CHECKPOINT_PATH):
+        chk = pd.read_csv(CHECKPOINT_PATH)
+        for _, row in chk.iterrows():
+            norm_k = normalize_name(row["artist_name"])
+            verified_map[norm_k] = row.to_dict()
+        print(f"Resuming with {len(verified_map):,} verified artists already preserved.")
+
+    # Populate preserved checkpoint rows into the dataframe
+    for idx in df.index:
+        norm_k = normalize_name(df.at[idx, "artist_name"])
+        if norm_k in verified_map:
+            record = verified_map[norm_k]
+            df.at[idx, "lastfm_listeners"] = record.get("lastfm_listeners")
+            df.at[idx, "lastfm_playcount"] = record.get("lastfm_playcount")
+            df.at[idx, "lastfm_url"] = record.get("lastfm_url")
+            df.at[idx, "catalog_intelligence_tier"] = "TIER_3_LASTFM_VERIFIED_ENTITY"
+
+    # Only process rows that have not yet been verified
+    unverified_mask = df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+    target_rows = [
+        (idx, str(df.at[idx, "artist_name"]))
+        for idx in df[unverified_mask].index
+    ]
+
+    total_to_process = len(target_rows)
+    print(f"Verifying {total_to_process:,} unverified artists with rate-safe spacing...\n")
+
+    if total_to_process == 0:
+        print("All artists already verified.")
+        return
+
+    session = requests.Session()
+    checkpoint_batch = []
+    completed_count = 0
+    new_verified = 0
+
+    with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:
+        future_to_idx = {
+            executor.submit(fetch_artist_lastfm, session, name): (idx, name)
+            for idx, name in target_rows
+        }
+
+        for future in as_completed(future_to_idx):
+            idx, name = future_to_idx[future]
+            completed_count += 1
+            time.sleep(0.35)
+
+            try:
+                result = future.result()
+            except Exception:
+                result = {"artist_name": name, "verified": False}
+
+            if result.get("verified"):
+                new_verified += 1
+                df.at[idx, "lastfm_listeners"] = result["lastfm_listeners"]
+                df.at[idx, "lastfm_playcount"] = result["lastfm_playcount"]
+                df.at[idx, "lastfm_url"] = result["lastfm_url"]
+                df.at[idx, "catalog_intelligence_tier"] = "TIER_3_LASTFM_VERIFIED_ENTITY"
+
+                checkpoint_batch.append({
+                    "artist_name": name,
+                    "lastfm_listeners": result["lastfm_listeners"],
+                    "lastfm_playcount": result["lastfm_playcount"],
+                    "lastfm_url": result["lastfm_url"],
+                    "catalog_intelligence_tier": "TIER_3_LASTFM_VERIFIED_ENTITY"
+                })
+
+            # Checkpoint every 50 newly verified artists
+            if len(checkpoint_batch) >= 50 or completed_count == total_to_process:
+                if checkpoint_batch:
+                    chk_df = pd.DataFrame(checkpoint_batch)
+                    chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=not os.path.exists(CHECKPOINT_PATH), index=False)
+                    checkpoint_batch = []
+                print(f"[{completed_count:,}/{total_to_process:,}] Checked | New Verified: {new_verified:,} (Total Verified: {len(verified_map) + new_verified:,})")
+
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nVerification complete! Saved to {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_lastfm_verification()

scripts/verify_artists_spotify_registry.py

@@ -0,0 +1,163 @@
+import os
+import re
+import time
+import base64
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+# Load variables directly from .env file
+load_dotenv()
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+CHECKPOINT_PATH = os.path.join(DATA_DIR, "checkpoints", "spotify_verification_checkpoint.csv")
+
+os.makedirs(os.path.join(DATA_DIR, "checkpoints"), exist_ok=True)
+
+SPOTIFY_CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
+SPOTIFY_CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")
+
+
+def get_spotify_token(client_id: str, client_secret: str) -> str:
+    auth_str = f"{client_id}:{client_secret}"
+    b64_auth = base64.b64encode(auth_str.encode()).decode()
+    resp = requests.post(
+        "https://accounts.spotify.com/api/token",
+        data={"grant_type": "client_credentials"},
+        headers={"Authorization": f"Basic {b64_auth}"},
+        timeout=15
+    )
+    if resp.status_code != 200:
+        raise ValueError(f"Spotify authentication failed (HTTP {resp.status_code}): {resp.text}")
+    return resp.json()["access_token"]
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def run_spotify_verification():
+    if not SPOTIFY_CLIENT_ID or not SPOTIFY_CLIENT_SECRET:
+        print("ERROR: SPOTIFY_CLIENT_ID or SPOTIFY_CLIENT_SECRET not found in .env.")
+        return
+
+    print("Authenticating with Spotify using .env credentials...")
+    token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
+    print("Authentication successful!")
+
+    headers = {"Authorization": f"Bearer {token}"}
+    session = requests.Session()
+
+    if not os.path.exists(INPUT_PATH):
+        print(f"ERROR: Target file not found at {INPUT_PATH}")
+        return
+
+    df = pd.read_csv(INPUT_PATH)
+
+    # Ensure required columns exist
+    for col in ["spotify_artist_id", "spotify_verified_followers", "spotify_profile_url"]:
+        if col not in df.columns:
+            df[col] = None
+
+    # Load existing progress from checkpoint
+    processed_artists = set()
+    if os.path.exists(CHECKPOINT_PATH):
+        chk = pd.read_csv(CHECKPOINT_PATH)
+        processed_artists = set(chk["artist_name"].dropna().apply(normalize_name))
+        print(f"Resuming: {len(processed_artists):,} artists already logged in checkpoint.")
+
+    # Target unverified tier records
+    unverified_mask = df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+    target_indices = [
+        idx for idx in df[unverified_mask].index
+        if normalize_name(df.at[idx, "artist_name"]) not in processed_artists
+    ]
+
+    print(f"Verifying {len(target_indices):,} catalog artists directly on Spotify...\n")
+
+    token_time = time.time()
+    checkpoint_records = []
+    matched_count = 0
+
+    for i, idx in enumerate(target_indices, start=1):
+        a_name = str(df.at[idx, "artist_name"])
+        target_norm = normalize_name(a_name)
+
+        # Refresh bearer token every 50 minutes (tokens expire after 60 minutes)
+        if time.time() - token_time > 3000:
+            token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
+            headers = {"Authorization": f"Bearer {token}"}
+            token_time = time.time()
+
+        url = f"https://api.spotify.com/v1/search?q={requests.utils.quote(a_name)}&type=artist&limit=5"
+
+        try:
+            resp = session.get(url, headers=headers, timeout=10)
+            if resp.status_code == 429:
+                retry_after = int(resp.headers.get("Retry-After", 5))
+                print(f"Rate limited. Pausing for {retry_after}s...")
+                time.sleep(retry_after)
+                resp = session.get(url, headers=headers, timeout=10)
+
+            if resp.status_code == 200:
+                items = resp.json().get("artists", {}).get("items", [])
+                matched_item = None
+
+                for item in items:
+                    cand_norm = normalize_name(item.get("name", ""))
+                    if cand_norm == target_norm:
+                        matched_item = item
+                        break
+
+                if matched_item:
+                    sp_id = matched_item["id"]
+                    followers = matched_item.get("followers", {}).get("total", 0)
+                    sp_url = matched_item.get("external_urls", {}).get("spotify", "")
+
+                    df.at[idx, "spotify_artist_id"] = sp_id
+                    df.at[idx, "spotify_verified_followers"] = followers
+                    df.at[idx, "spotify_profile_url"] = sp_url
+                    df.at[idx, "catalog_intelligence_tier"] = "TIER_3_SPOTIFY_VERIFIED_ENTITY"
+                    matched_count += 1
+
+            checkpoint_records.append({
+                "artist_name": a_name,
+                "spotify_artist_id": df.at[idx, "spotify_artist_id"],
+                "spotify_verified_followers": df.at[idx, "spotify_verified_followers"],
+                "spotify_profile_url": df.at[idx, "spotify_profile_url"],
+                "catalog_intelligence_tier": df.at[idx, "catalog_intelligence_tier"]
+            })
+
+        except Exception as e:
+            pass
+
+        # Checkpoint every 100 artists
+        if i % 100 == 0:
+            chk_df = pd.DataFrame(checkpoint_records)
+            if os.path.exists(CHECKPOINT_PATH):
+                chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
+            else:
+                chk_df.to_csv(CHECKPOINT_PATH, index=False)
+            checkpoint_records = []
+            print(f"[{i:,}/{len(target_indices):,}] Checked | Verified on Spotify: {matched_count:,}")
+
+    # Write remaining checkpoint records
+    if checkpoint_records:
+        chk_df = pd.DataFrame(checkpoint_records)
+        if os.path.exists(CHECKPOINT_PATH):
+            chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
+        else:
+            chk_df.to_csv(CHECKPOINT_PATH, index=False)
+
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nVerification complete! Final catalog saved to: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_spotify_verification()