feat(cohort): reconcile 500-artist telemetry, update avatars, and add drift detection
- Decouple commercial CSPC sales columns from the 500 cohort telemetry feed - Synchronize 100% canonical platform URLs (YouTube, Spotify, Instagram) from hiffi_catalog.db to hiffi_multiplatform_500.csv - Enforce Tier-1 zero-imputation policy for Hak (@haknyc) with RFC 8259-compliant nulls - Reconcile and crop portrait avatar assets across cohort entities - Implement automated drift detector and velocity circuit breaker (drift_detector.py)
This commit is contained in:
Srilekha committed
1 parent
df5b4a0424
commit
ebba583c66
52 files changed
+1864
-2275
No files matched your search
@@ -1,9 +1,9 @@
|
||||
"""
|
||||
"""
|
||||
app.py
|
||||
HIFFI Artist Intelligence - Multi-Platform Executive Dashboard
|
||||
Dual-Engine Architecture:
|
||||
1. HIFFI Hip-Hop 500: Multi-platform evidence tiers, YouTube/Spotify/IG telemetry, BVI, CPI engine.
|
||||
2. Global Master Catalog (17k): Empirical registry verification (CSPC, RIAA, Last.fm) with ordinal tier ranking.
|
||||
Dual-Engine Production Architecture:
|
||||
1. HIFFI Hip-Hop 500: Pure Digital Telemetry (45% Spotify, 35% YouTube, 20% Instagram), Evidence Tiers, BVI.
|
||||
2. Global Master Catalog (17k): Empirical registry verification (CSPC, RIAA, Last.fm).
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
@@ -13,16 +13,6 @@ import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import streamlit as st
|
||||
|
||||
try:
|
||||
from core.cpi_engine import CommercialPowerEngine
|
||||
except ImportError:
|
||||
CommercialPowerEngine = None
|
||||
|
||||
try:
|
||||
from exporters.excel_workbook import generate_executive_sales_workbook
|
||||
except ImportError:
|
||||
generate_executive_sales_workbook = None
|
||||
|
||||
try:
|
||||
from core.ranking_engine import HIFFIRankingEngine
|
||||
except ImportError:
|
||||
@@ -40,7 +30,6 @@ PATH_500 = Path("data/hiffi_multiplatform_500.csv")
|
||||
PATH_17K = Path("data/master_artists_intelligence_17k_final.csv")
|
||||
FALLBACK_PATH = Path("enriched_artists.csv")
|
||||
|
||||
# Global CSS Styling
|
||||
st.markdown(
|
||||
"""
|
||||
<style>
|
||||
@@ -75,35 +64,6 @@ st.markdown(
|
||||
font-size: 0.72rem;
|
||||
color: #6b7280;
|
||||
}
|
||||
.coverage-card {
|
||||
background: #0f172a;
|
||||
border: 1px solid #1e293b;
|
||||
border-radius: 8px;
|
||||
padding: 12px 18px;
|
||||
margin-bottom: 16px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.sim-kpi-card {
|
||||
background: #0f172a;
|
||||
border: 1px solid #334155;
|
||||
border-radius: 8px;
|
||||
padding: 10px 14px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.sim-kpi-title {
|
||||
font-size: 0.68rem;
|
||||
font-weight: 700;
|
||||
color: #94a3b8;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
.sim-kpi-val {
|
||||
font-size: 1.25rem;
|
||||
font-weight: 800;
|
||||
margin-top: 2px;
|
||||
}
|
||||
.podium-tile {
|
||||
background: #111827;
|
||||
border-radius: 12px;
|
||||
@@ -125,7 +85,6 @@ st.markdown(
|
||||
.badge-tier3 { background: rgba(16, 185, 129, 0.2); color: #10b981; }
|
||||
.badge-tier2 { background: rgba(56, 189, 248, 0.2); color: #38bdf8; }
|
||||
.badge-tier1 { background: rgba(245, 158, 11, 0.2); color: #f59e0b; }
|
||||
.badge-dormant { background: rgba(239, 68, 68, 0.2); color: #ef4444; }
|
||||
.custom-divider {
|
||||
height: 1px;
|
||||
background: #1f2937;
|
||||
@@ -137,10 +96,8 @@ st.markdown(
|
||||
)
|
||||
|
||||
|
||||
@st.cache_data(ttl=60)
|
||||
def load_hiffi_500():
|
||||
df = pd.DataFrame()
|
||||
|
||||
if PATH_DB.exists():
|
||||
try:
|
||||
conn = sqlite3.connect(PATH_DB)
|
||||
@@ -148,51 +105,35 @@ def load_hiffi_500():
|
||||
WITH latest_telemetry AS (
|
||||
SELECT t.*, ROW_NUMBER() OVER (
|
||||
PARTITION BY t.artist_id
|
||||
ORDER BY t.last_synced_at DESC, t.telemetry_id DESC
|
||||
ORDER BY t.last_synced_at DESC, t.rowid DESC
|
||||
) as rn
|
||||
FROM telemetry_metrics t
|
||||
),
|
||||
latest_rankings AS (
|
||||
SELECT r.*, ROW_NUMBER() OVER (
|
||||
PARTITION BY r.artist_id
|
||||
ORDER BY r.ranking_id DESC
|
||||
ORDER BY r.rowid DESC
|
||||
) as rn
|
||||
FROM rankings r
|
||||
),
|
||||
latest_commercial AS (
|
||||
SELECT c.*, ROW_NUMBER() OVER (
|
||||
PARTITION BY c.artist_id
|
||||
ORDER BY c.commercial_id DESC
|
||||
) as rn
|
||||
FROM commercial_sales c
|
||||
)
|
||||
SELECT
|
||||
a.artist_id,
|
||||
a.artist_name as name,
|
||||
a.image_verified_flag,
|
||||
r.catalog_rank as hiffi_xp_published_rank,
|
||||
r.tier_rank,
|
||||
r.catalog_intelligence_tier,
|
||||
t.subscribers_count as subscriber_count,
|
||||
t.total_views as lifetime_views,
|
||||
t.spotify_monthly_listeners,
|
||||
t.instagram_followers,
|
||||
t.daily_view_velocity,
|
||||
t.growth_velocity_7d as momentum_7d,
|
||||
t.growth_velocity_30d as momentum_30d,
|
||||
t.spotify_daily_change,
|
||||
t.momentum_score,
|
||||
c.pure_album_sales,
|
||||
c.physical_singles_sales,
|
||||
c.digital_singles_sales,
|
||||
c.streaming_eas,
|
||||
c.total_eas_cspc,
|
||||
c.sales_data_status
|
||||
FROM artists a
|
||||
COALESCE(t.subscribers_count, 0) as subscriber_count,
|
||||
COALESCE(t.total_views, 0) as lifetime_views,
|
||||
COALESCE(t.spotify_monthly_listeners, 0) as spotify_monthly_listeners,
|
||||
COALESCE(t.instagram_followers, 0) as instagram_followers,
|
||||
COALESCE(t.daily_view_velocity, 0) as daily_view_velocity,
|
||||
COALESCE(t.growth_velocity_7d, 0.0) as momentum_7d,
|
||||
COALESCE(t.growth_velocity_30d, 0.0) as momentum_30d,
|
||||
COALESCE(t.spotify_daily_change, 0) as spotify_daily_change
|
||||
FROM rankings r
|
||||
JOIN artists a ON r.artist_id = a.artist_id
|
||||
LEFT JOIN latest_telemetry t ON a.artist_id = t.artist_id AND t.rn = 1
|
||||
LEFT JOIN latest_rankings r ON a.artist_id = r.artist_id AND r.rn = 1
|
||||
LEFT JOIN latest_commercial c ON a.artist_id = c.artist_id AND c.rn = 1
|
||||
WHERE a.image_verified_flag = 1
|
||||
WHERE r.catalog_rank <= 500
|
||||
ORDER BY r.catalog_rank ASC;
|
||||
"""
|
||||
df = pd.read_sql_query(query, conn)
|
||||
conn.close()
|
||||
@@ -202,7 +143,10 @@ def load_hiffi_500():
|
||||
if df.empty:
|
||||
target = PATH_500 if PATH_500.exists() else FALLBACK_PATH
|
||||
if target.exists():
|
||||
try:
|
||||
df = pd.read_csv(target)
|
||||
except Exception:
|
||||
df = pd.DataFrame()
|
||||
|
||||
if df.empty:
|
||||
return pd.DataFrame()
|
||||
@@ -212,77 +156,71 @@ def load_hiffi_500():
|
||||
|
||||
if "subscriber_count" not in df.columns and "subscribers_count" in df.columns:
|
||||
df["subscriber_count"] = df["subscribers_count"]
|
||||
elif "subscriber_count" not in df.columns:
|
||||
df["subscriber_count"] = 0
|
||||
|
||||
if "lifetime_views" not in df.columns and "total_views" in df.columns:
|
||||
df["lifetime_views"] = df["total_views"]
|
||||
elif "lifetime_views" not in df.columns:
|
||||
df["lifetime_views"] = 0
|
||||
|
||||
if "spotify_monthly_listeners" not in df.columns:
|
||||
df["spotify_monthly_listeners"] = 0
|
||||
|
||||
if "instagram_followers" not in df.columns:
|
||||
df["instagram_followers"] = 0
|
||||
|
||||
if "hiffi_xp_published_rank" not in df.columns:
|
||||
df["hiffi_xp_published_rank"] = df.get("catalog_rank", df.get("hiffi_xp_rank", df.get("hiffi_rank", df.index + 1)))
|
||||
df["hiffi_xp_published_rank"] = df.get("catalog_rank", df.get("hiffi_xp_rank", df.index + 1))
|
||||
|
||||
has_yt = df.get("subscriber_count", pd.Series(index=df.index)).fillna(0).gt(0).astype(int)
|
||||
has_sp = df.get("spotify_monthly_listeners", pd.Series(index=df.index)).fillna(0).gt(0).astype(int)
|
||||
has_ig = df.get("instagram_followers", pd.Series(index=df.index)).fillna(0).gt(0).astype(int)
|
||||
df["platform_evidence_tier"] = has_yt + has_sp + has_ig
|
||||
has_sp = pd.to_numeric(df["spotify_monthly_listeners"], errors="coerce").fillna(0).gt(0).astype(int)
|
||||
has_yt = pd.to_numeric(df["subscriber_count"], errors="coerce").fillna(0).gt(0).astype(int)
|
||||
has_ig = pd.to_numeric(df["instagram_followers"], errors="coerce").fillna(0).gt(0).astype(int)
|
||||
df["platform_evidence_tier"] = has_sp + has_yt + has_ig
|
||||
|
||||
numeric_cols = [
|
||||
"hiffi_xp_published_rank", "hiffi_xp_rank", "hiffi_xp_score", "production_hiffi_rank",
|
||||
"production_hiffi_score", "platform_evidence_tier", "breakout_velocity_index",
|
||||
"subscriber_count", "daily_view_velocity", "momentum_30d", "spotify_monthly_listeners",
|
||||
"spotify_daily_change", "instagram_followers", "confidence_score", "breakout_rank",
|
||||
"chartmasters_sales_rank", "cohort_sales_rank", "pure_album_sales",
|
||||
"physical_singles_sales", "digital_singles_sales", "streaming_eas", "total_eas_cspc",
|
||||
"riaa_certified_album_units", "riaa_certified_singles_units"
|
||||
"hiffi_xp_published_rank", "hiffi_xp_rank", "hiffi_xp_score", "platform_evidence_tier",
|
||||
"subscriber_count", "lifetime_views", "spotify_monthly_listeners", "instagram_followers",
|
||||
"daily_view_velocity", "momentum_7d", "momentum_30d", "breakout_velocity_index"
|
||||
]
|
||||
for col in numeric_cols:
|
||||
if col in df.columns:
|
||||
df[col] = pd.to_numeric(df[col], errors="coerce")
|
||||
|
||||
# Dynamic BVI computation
|
||||
if HIFFIRankingEngine is not None and ("breakout_velocity_index" not in df.columns or df["breakout_velocity_index"].isna().all()):
|
||||
try:
|
||||
eng_rank = HIFFIRankingEngine()
|
||||
temp_df = df.copy()
|
||||
if "status_flag" not in temp_df.columns:
|
||||
temp_df["status_flag"] = "COMPLETE"
|
||||
df["breakout_velocity_index"] = eng_rank.calculate_breakout_velocity(temp_df)
|
||||
eng = HIFFIRankingEngine()
|
||||
df["breakout_velocity_index"] = eng.calculate_breakout_velocity(df)
|
||||
except Exception:
|
||||
df["breakout_velocity_index"] = 0.0
|
||||
|
||||
if "breakout_velocity_index" not in df.columns:
|
||||
df["breakout_velocity_index"] = 0.0
|
||||
|
||||
# Calculate HIFFI-XP composite if not present
|
||||
if "hiffi_xp_score" not in df.columns or df["hiffi_xp_score"].isna().all():
|
||||
if HIFFIRankingEngine is not None:
|
||||
try:
|
||||
eng_rank = HIFFIRankingEngine()
|
||||
temp_df = df.copy()
|
||||
if "status_flag" not in temp_df.columns:
|
||||
temp_df["status_flag"] = "COMPLETE"
|
||||
temp_df = eng_rank.score_and_rank(temp_df)
|
||||
temp_df = eng_rank.calculate_hiffi_xp(temp_df)
|
||||
df["hiffi_xp_score"] = temp_df["hiffi_xp_score"]
|
||||
df["effective_weights"] = temp_df.get("effective_weights", "yt:50.0%,sp_monthly:25.0%,sp_velocity:5.0%,ig:20.0%")
|
||||
if "production_hiffi_score" not in df.columns:
|
||||
df["production_hiffi_score"] = temp_df.get("production_hiffi_score", 0.0)
|
||||
eng = HIFFIRankingEngine()
|
||||
ranked_df = eng.calculate_hiffi_xp(df)
|
||||
df["hiffi_xp_score"] = ranked_df["hiffi_xp_score"]
|
||||
df["effective_weights"] = ranked_df.get("effective_weights", "sp:45.0%,yt:35.0%,ig:20.0%")
|
||||
df["catalog_rank"] = ranked_df.get("catalog_rank", df["hiffi_xp_published_rank"])
|
||||
df["hiffi_xp_published_rank"] = df["catalog_rank"]
|
||||
except Exception:
|
||||
df["hiffi_xp_score"] = 50.0
|
||||
sp_n = df["spotify_monthly_listeners"].fillna(0) / max(df["spotify_monthly_listeners"].max(), 1)
|
||||
yt_n = df["subscriber_count"].fillna(0) / max(df["subscriber_count"].max(), 1)
|
||||
ig_n = df["instagram_followers"].fillna(0) / max(df["instagram_followers"].max(), 1)
|
||||
df["hiffi_xp_score"] = (sp_n * 45.0 + yt_n * 35.0 + ig_n * 20.0).round(2)
|
||||
df["effective_weights"] = "sp:45.0%,yt:35.0%,ig:20.0%"
|
||||
else:
|
||||
df["hiffi_xp_score"] = 50.0
|
||||
|
||||
if "effective_weights" not in df.columns:
|
||||
df["effective_weights"] = "yt:50.0%,sp_monthly:25.0%,sp_velocity:5.0%,ig:20.0%"
|
||||
sp_n = df["spotify_monthly_listeners"].fillna(0) / max(df["spotify_monthly_listeners"].max(), 1)
|
||||
yt_n = df["subscriber_count"].fillna(0) / max(df["subscriber_count"].max(), 1)
|
||||
ig_n = df["instagram_followers"].fillna(0) / max(df["instagram_followers"].max(), 1)
|
||||
df["hiffi_xp_score"] = (sp_n * 45.0 + yt_n * 35.0 + ig_n * 20.0).round(2)
|
||||
df["effective_weights"] = "sp:45.0%,yt:35.0%,ig:20.0%"
|
||||
|
||||
if "anomaly_flags" not in df.columns:
|
||||
df["anomaly_flags"] = "CLEAN"
|
||||
|
||||
if CommercialPowerEngine is not None:
|
||||
try:
|
||||
cpi_eng = CommercialPowerEngine()
|
||||
df["commercial_power_index"] = cpi_eng.calculate_cpi(df)
|
||||
except Exception:
|
||||
df["commercial_power_index"] = None
|
||||
|
||||
if "subgenre" not in df.columns:
|
||||
def infer_subgenre(r):
|
||||
subs = r.get("subscriber_count", 0) or 0
|
||||
@@ -298,7 +236,6 @@ def load_hiffi_500():
|
||||
return df
|
||||
|
||||
|
||||
@st.cache_data(ttl=60)
|
||||
def load_master_17k():
|
||||
if not PATH_17K.exists():
|
||||
return pd.DataFrame()
|
||||
@@ -310,74 +247,54 @@ def load_master_17k():
|
||||
|
||||
df_500 = load_hiffi_500()
|
||||
df_17k = load_master_17k()
|
||||
engine = CommercialPowerEngine() if CommercialPowerEngine is not None else None
|
||||
|
||||
|
||||
def get_provenance_badge(status: str) -> str:
|
||||
s = str(status).strip()
|
||||
if "CSPC" in s or s == "VERIFIED_CSPC":
|
||||
return "✅ VERIFIED (CSPC)"
|
||||
elif "RIAA" in s or "COMMERCIAL_CORE" in s:
|
||||
return "🏅 VERIFIED (RIAA)"
|
||||
elif "LASTFM" in s:
|
||||
return "📻 VERIFIED (Last.fm)"
|
||||
elif "ESTIMATED_SEA" in s:
|
||||
return "📊 ESTIMATED (SEA)"
|
||||
return "⏳ UNAUDITED / PENDING"
|
||||
|
||||
|
||||
# ----------------- Sidebar Controls -----------------
|
||||
with st.sidebar:
|
||||
st.markdown("### ⚡ HIFFI Intelligence Suite")
|
||||
st.caption("Hip-Hop 500 & Master Catalog Intelligence")
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
catalog_source = st.selectbox(
|
||||
"🗂️ Active Data Scope:",
|
||||
["⚡ Hip-Hop 500 (Executive Telemetry)", "🌐 Global Master Catalog (17,000+ Acts)"],
|
||||
"Active Data Scope:",
|
||||
["Hip-Hop 500 (Executive Telemetry)", "Global Master Catalog (17,000+ Acts)"],
|
||||
index=0,
|
||||
)
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
# =====================================================================
|
||||
# MODULE 1: GLOBAL MASTER CATALOG (17,000+ ACTS)
|
||||
# =====================================================================
|
||||
if "Global Master Catalog" in catalog_source:
|
||||
with st.sidebar:
|
||||
st.markdown("##### 🔍 17k Catalog Filters")
|
||||
st.markdown("##### 17k Catalog Filters")
|
||||
if df_17k.empty:
|
||||
st.error("Master 17k dataset not found at data/master_artists_intelligence_17k_final.csv")
|
||||
st.stop()
|
||||
|
||||
tier_list = ["All Tiers"] + sorted(df_17k["catalog_intelligence_tier"].dropna().unique().tolist())
|
||||
tier_list = ["All Tiers"] + sorted(df_17k.get("catalog_intelligence_tier", pd.Series(dtype=str)).dropna().unique().tolist())
|
||||
sel_tier = st.selectbox("Intelligence Tier:", tier_list, index=0)
|
||||
search_17k = st.text_input("Search Artist Name:", placeholder="e.g. Eminem, The Roots...")
|
||||
|
||||
filt_17k = df_17k.copy()
|
||||
if sel_tier != "All Tiers":
|
||||
if sel_tier != "All Tiers" and "catalog_intelligence_tier" in filt_17k.columns:
|
||||
filt_17k = filt_17k[filt_17k["catalog_intelligence_tier"] == sel_tier]
|
||||
if search_17k:
|
||||
if search_17k and "name" in filt_17k.columns:
|
||||
filt_17k = filt_17k[filt_17k["name"].str.contains(search_17k, case=False, na=False)]
|
||||
|
||||
head1, head2 = st.columns([3, 1])
|
||||
with head1:
|
||||
st.markdown("<h2 style='margin-bottom: 2px;'>🌐 Global Master Artist Catalog (17k)</h2>", unsafe_allow_html=True)
|
||||
st.markdown("<h2 style='margin-bottom: 2px;'>Global Master Artist Catalog (17k)</h2>", unsafe_allow_html=True)
|
||||
st.markdown("<p style='color: #9ca3af; font-size: 0.92rem;'>Empirical registry validation via CSPC sales, RIAA certifications, and verified Last.fm global scrobbles.</p>", unsafe_allow_html=True)
|
||||
with head2:
|
||||
st.markdown(f"<div style='text-align: right; color: #6b7280; font-size: 0.8rem; padding-top: 10px;'>Filtered: <b>{len(filt_17k):,} / {len(df_17k):,}</b> acts</div>", unsafe_allow_html=True)
|
||||
|
||||
# 17k Macro KPIs
|
||||
mk1, mk2, mk3, mk4, mk5 = st.columns(5)
|
||||
with mk1:
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title">Universe Size</div><div class="kpi-value">{len(df_17k):,}</div><div class="kpi-subtitle">Total Cataloged</div></div>', unsafe_allow_html=True)
|
||||
with mk2:
|
||||
ver_cnt = df_17k["catalog_rank"].notna().sum()
|
||||
ver_cnt = df_17k["catalog_rank"].notna().sum() if "catalog_rank" in df_17k.columns else 0
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #10b981;">Verified Entities</div><div class="kpi-value" style="color: #10b981;">{ver_cnt:,}</div><div class="kpi-subtitle">Registry Authenticated</div></div>', unsafe_allow_html=True)
|
||||
with mk3:
|
||||
t1_cnt = (df_17k["catalog_intelligence_tier"] == "TIER_1_GLOBAL_SUPERSTAR_CSPC").sum()
|
||||
t1_cnt = (df_17k.get("catalog_intelligence_tier") == "TIER_1_GLOBAL_SUPERSTAR_CSPC").sum()
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #f59e0b;">Tier 1 Superstars</div><div class="kpi-value" style="color: #f59e0b;">{t1_cnt:,}</div><div class="kpi-subtitle">Audited CSPC EAS</div></div>', unsafe_allow_html=True)
|
||||
with mk4:
|
||||
lfm_cnt = (df_17k["catalog_intelligence_tier"] == "TIER_3_LASTFM_VERIFIED_ENTITY").sum()
|
||||
lfm_cnt = (df_17k.get("catalog_intelligence_tier") == "TIER_3_LASTFM_VERIFIED_ENTITY").sum()
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #38bdf8;">Last.fm Verified</div><div class="kpi-value" style="color: #38bdf8;">{lfm_cnt:,}</div><div class="kpi-subtitle">Active Platform Entities</div></div>', unsafe_allow_html=True)
|
||||
with mk5:
|
||||
unver_cnt = len(df_17k) - ver_cnt
|
||||
@@ -393,7 +310,7 @@ if "Global Master Catalog" in catalog_source:
|
||||
|
||||
st.dataframe(
|
||||
filt_17k[disp_cols].reset_index(drop=True),
|
||||
use_container_width=True,
|
||||
width="stretch",
|
||||
height=560,
|
||||
column_config={
|
||||
"catalog_rank": st.column_config.NumberColumn("Global Rank", format="#%d", width="small"),
|
||||
@@ -408,389 +325,157 @@ if "Global Master Catalog" in catalog_source:
|
||||
)
|
||||
|
||||
with st.sidebar:
|
||||
st.markdown("##### 💾 17k Data Export")
|
||||
st.markdown("##### 17k Data Export")
|
||||
csv_17k = filt_17k.to_csv(index=False).encode("utf-8")
|
||||
st.download_button("📥 Export 17k Table CSV", data=csv_17k, file_name="master_17k_catalog.csv", mime="text/csv", use_container_width=True)
|
||||
st.download_button("Export 17k Table CSV", data=csv_17k, file_name="master_17k_catalog.csv", mime="text/csv", width="stretch")
|
||||
|
||||
# =====================================================================
|
||||
# MODULE 2: HIFFI HIP-HOP 500 (EXECUTIVE DASHBOARD)
|
||||
# =====================================================================
|
||||
else:
|
||||
df = df_500
|
||||
if df.empty:
|
||||
st.error("Hip-Hop 500 dataset not found. Please verify data/hiffi_catalog.db exists.")
|
||||
st.error("Hip-Hop 500 dataset not found in data/hiffi_catalog.db")
|
||||
st.stop()
|
||||
|
||||
with st.sidebar:
|
||||
view_mode = st.radio(
|
||||
"Leaderboard Perspective:",
|
||||
[
|
||||
"🏆 Published Multi-Platform Rank",
|
||||
"🚀 Breakout Velocity (BVI)",
|
||||
"💿 Record Sales (CSPC Methodology)",
|
||||
"🧪 Commercial Simulation Engine (CPI)",
|
||||
"🔍 Full Cohort Audit Grid",
|
||||
"Published Multi-Platform Rank",
|
||||
"Breakout Velocity (BVI)",
|
||||
"Full Cohort Audit Grid",
|
||||
],
|
||||
index=0,
|
||||
)
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
catalog_filter = "All Monitored Artists (500)"
|
||||
if view_mode in ("💿 Record Sales (CSPC Methodology)", "🧪 Commercial Simulation Engine (CPI)"):
|
||||
st.markdown("##### 🔍 Sales Catalog Scope")
|
||||
catalog_filter = st.radio(
|
||||
"Filter Rows by Sales Telemetry:",
|
||||
["All Monitored Artists (500)", "Verified Only (321)", "Pure CSPC Only (62)"],
|
||||
index=0,
|
||||
)
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
sim_weight = 0.15
|
||||
sim_mode = "matched_only"
|
||||
if view_mode == "🧪 Commercial Simulation Engine (CPI)" and engine is not None:
|
||||
st.markdown("##### 🧪 Simulation Parameters")
|
||||
sim_mode_choice = st.radio(
|
||||
"Evaluation Scope:",
|
||||
["🎯 Matched Cohort Only", "🌐 Global Cohort (N=500, Weight Redistribution)"],
|
||||
index=1 if catalog_filter == "All Monitored Artists (500)" else 0,
|
||||
)
|
||||
sim_mode = "matched_only" if "Matched" in sim_mode_choice else "global_redistribution"
|
||||
sim_weight = st.select_slider(
|
||||
"Commercial Signal Weight:",
|
||||
options=[0.0, 0.05, 0.10, 0.15, 0.20],
|
||||
value=0.15,
|
||||
format_func=lambda x: f"{int(x*100)}% Commercial",
|
||||
)
|
||||
st.caption(f"Baseline Score Weight: **{int((1.0 - sim_weight)*100)}%**")
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
st.markdown("##### 🛡️ Cohort Filters")
|
||||
st.markdown("##### Cohort Filters")
|
||||
all_verticals = sorted(df["subgenre"].dropna().unique().tolist())
|
||||
selected_verticals = st.multiselect("Vertical", options=all_verticals, default=all_verticals)
|
||||
selected_verticals = st.multiselect("Vertical", options=all_verticals, default=all_verticals, key="v_sel_pure")
|
||||
|
||||
tier_options = sorted(df["platform_evidence_tier"].dropna().unique().astype(int).tolist(), reverse=True)
|
||||
selected_tiers = st.multiselect("Evidence Tier", options=tier_options, default=tier_options)
|
||||
selected_tiers = st.multiselect("Evidence Tier", options=tier_options, default=tier_options, key="t_sel_pure")
|
||||
min_xp = st.slider("Min HIFFI-XP Score", 0.0, 100.0, 0.0, 5.0)
|
||||
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
st.markdown("##### Export")
|
||||
csv_payload = df.to_csv(index=False).encode("utf-8")
|
||||
st.download_button(
|
||||
"Download Top 500 CSV",
|
||||
data=csv_payload,
|
||||
file_name="hiffi_top500_telemetry.csv",
|
||||
mime="text/csv",
|
||||
width="stretch",
|
||||
)
|
||||
st.caption("Telemetry Engine: 45% Spotify - 35% YouTube - 20% Instagram")
|
||||
|
||||
filtered_df = df[
|
||||
(df["subgenre"].isin(selected_verticals))
|
||||
& (df["platform_evidence_tier"].isin(selected_tiers))
|
||||
& (df["hiffi_xp_score"].fillna(0.0) >= min_xp)
|
||||
].copy()
|
||||
|
||||
if "sales_data_status" in filtered_df.columns:
|
||||
if catalog_filter == "Pure CSPC Only (62)":
|
||||
filtered_df = filtered_df[filtered_df["sales_data_status"] == "VERIFIED_CSPC"].copy()
|
||||
elif catalog_filter == "Verified Only (321)":
|
||||
filtered_df = filtered_df[filtered_df["sales_data_status"].isin(["VERIFIED_CSPC", "VERIFIED_RIAA"])].copy()
|
||||
|
||||
sim_metrics = {}
|
||||
if view_mode == "🧪 Commercial Simulation Engine (CPI)" and engine is not None:
|
||||
try:
|
||||
filtered_df = engine.simulate_ranking(filtered_df, commercial_weight=sim_weight, mode=sim_mode)
|
||||
sim_metrics = engine.compute_simulation_metrics(filtered_df)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
with st.sidebar:
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
st.markdown("##### 💾 Unified Audit Exports")
|
||||
|
||||
if generate_executive_sales_workbook is not None:
|
||||
try:
|
||||
excel_workbook_bytes = generate_executive_sales_workbook(filtered_df, commercial_weight=sim_weight, mode=sim_mode)
|
||||
st.download_button(
|
||||
"📊 Multi-Tab Audit Excel (.xlsx)",
|
||||
data=excel_workbook_bytes,
|
||||
file_name="hiffi_executive_sales_audit.xlsx",
|
||||
mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
use_container_width=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
csv_payload = filtered_df.to_csv(index=False).encode("utf-8")
|
||||
st.download_button(
|
||||
"📥 Filtered Table CSV",
|
||||
data=csv_payload,
|
||||
file_name="hiffi_multiplatform_filtered.csv",
|
||||
mime="text/csv",
|
||||
use_container_width=True,
|
||||
)
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
st.caption("Governance: CSPC Methodology • EAS Units • Zero-Imputation Compliant")
|
||||
|
||||
head_col1, head_col2 = st.columns([3, 1])
|
||||
with head_col1:
|
||||
st.markdown("<h2 style='margin-bottom: 2px;'>HIFFI Hip-Hop 500 Intelligence Hub</h2>", unsafe_allow_html=True)
|
||||
st.markdown("<p style='color: #9ca3af; font-size: 0.92rem; margin-top: 0;'>Multi-platform composite rankings with zero-imputation policy, dynamic weight redistribution, and verified record sales.</p>", unsafe_allow_html=True)
|
||||
st.markdown("<p style='color: #9ca3af; font-size: 0.90rem; margin-top: 0;'>Pure multi-platform composite rankings (45% Spotify, 35% YouTube, 20% Instagram) with zero-imputation policy.</p>", unsafe_allow_html=True)
|
||||
with head_col2:
|
||||
st.markdown(
|
||||
f"<div style='text-align: right; color: #6b7280; font-size: 0.8rem; padding-top: 10px;'>Filtered Cohort: <b>{len(filtered_df)} / {len(df)}</b> artists</div>",
|
||||
f"<div style='text-align: right; color: #6b7280; font-size: 0.85rem; padding-top: 10px;'>Active Cohort: <b>{len(filtered_df)} / {len(df)}</b> artists</div>",
|
||||
unsafe_allow_html=True,
|
||||
)
|
||||
|
||||
# 500 Macro KPI Cards (6 Columns with YouTube Telemetry)
|
||||
k1, k2, k3, k4, k5, k6 = st.columns(6)
|
||||
k1, k2, k3, k4, k5 = st.columns(5)
|
||||
with k1:
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title">Tracked Artists</div><div class="kpi-value">{len(df)}</div><div class="kpi-subtitle">Monitored Cohort</div></div>', unsafe_allow_html=True)
|
||||
with k2:
|
||||
t3_count = len(df[df["platform_evidence_tier"] == 3])
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #10b981;">Tier 3 (Complete)</div><div class="kpi-value" style="color: #10b981;">{t3_count}</div><div class="kpi-subtitle">YouTube + Spotify + Instagram</div></div>', unsafe_allow_html=True)
|
||||
t3_count = int((df["platform_evidence_tier"] == 3).sum())
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #10b981;">Tier 3 (Complete)</div><div class="kpi-value" style="color: #10b981;">{t3_count}</div><div class="kpi-subtitle">Spotify + YT + IG</div></div>', unsafe_allow_html=True)
|
||||
with k3:
|
||||
yt_count = df["subscriber_count"].fillna(0).gt(0).sum() if "subscriber_count" in df.columns else len(df)
|
||||
yt_count = int(df["subscriber_count"].fillna(0).gt(0).sum())
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #ef4444;">YouTube Telemetry</div><div class="kpi-value" style="color: #ef4444;">{yt_count}</div><div class="kpi-subtitle">Verified video channels</div></div>', unsafe_allow_html=True)
|
||||
with k4:
|
||||
sp_count = df["spotify_monthly_listeners"].fillna(0).gt(0).sum() if "spotify_monthly_listeners" in df.columns else len(df)
|
||||
sp_count = int(df["spotify_monthly_listeners"].fillna(0).gt(0).sum())
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #38bdf8;">Spotify Telemetry</div><div class="kpi-value" style="color: #38bdf8;">{sp_count}</div><div class="kpi-subtitle">Verified monthly listeners</div></div>', unsafe_allow_html=True)
|
||||
with k5:
|
||||
ig_count = df["instagram_followers"].fillna(0).gt(0).sum() if "instagram_followers" in df.columns else len(df)
|
||||
ig_count = int(df["instagram_followers"].fillna(0).gt(0).sum())
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #f59e0b;">Instagram Audience</div><div class="kpi-value" style="color: #f59e0b;">{ig_count}</div><div class="kpi-subtitle">Verified public followings</div></div>', unsafe_allow_html=True)
|
||||
with k6:
|
||||
cataloged_count = df["total_eas_cspc"].fillna(0).gt(0).sum() if "total_eas_cspc" in df.columns else 0
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #ec4899;">Catalog Coverage</div><div class="kpi-value" style="color: #ec4899;">{cataloged_count}</div><div class="kpi-subtitle">Commercial Units Indexed</div></div>', unsafe_allow_html=True)
|
||||
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
if view_mode in ("💿 Record Sales (CSPC Methodology)", "🧪 Commercial Simulation Engine (CPI)") and "sales_data_status" in df.columns:
|
||||
verified_cspc_cnt = int((df["sales_data_status"] == "VERIFIED_CSPC").sum())
|
||||
verified_riaa_cnt = int((df["sales_data_status"] == "VERIFIED_RIAA").sum())
|
||||
estimated_sea_cnt = int((df["sales_data_status"] == "ESTIMATED_SEA").sum())
|
||||
total_active_coverage = verified_cspc_cnt + verified_riaa_cnt + estimated_sea_cnt
|
||||
coverage_pct = (total_active_coverage / len(df)) * 100.0 if len(df) > 0 else 0.0
|
||||
|
||||
st.markdown(
|
||||
f"""
|
||||
<div class="coverage-card">
|
||||
<div>
|
||||
<span style="font-weight: 800; color: #f9fafb; font-size: 0.92rem;">📡 Commercial Telemetry Coverage Progress:</span>
|
||||
<span style="color: #10b981; font-weight: 700; margin-left: 8px;">{verified_cspc_cnt} CSPC Verified</span>
|
||||
<span style="color: #64748b; margin: 0 6px;">•</span>
|
||||
<span style="color: #38bdf8; font-weight: 700;">{verified_riaa_cnt} RIAA Certified</span>
|
||||
<span style="color: #64748b; margin: 0 6px;">•</span>
|
||||
<span style="color: #a855f7; font-weight: 600;">{estimated_sea_cnt} Streaming SEA</span>
|
||||
</div>
|
||||
<div style="font-size: 0.85rem; color: #94a3b8;">
|
||||
Active Commercial Coverage: <strong style="color: #10b981;">{coverage_pct:.1f}%</strong> ({total_active_coverage}/{len(df)})
|
||||
</div>
|
||||
</div>
|
||||
""",
|
||||
unsafe_allow_html=True,
|
||||
)
|
||||
|
||||
if view_mode == "💿 Record Sales (CSPC Methodology)":
|
||||
st.markdown("### 💿 Global Record Sales & Lifetime Commercial Ranking")
|
||||
st.caption("Standardized on ChartMasters CSPC, verified RIAA certifications, and video/audio streaming equivalent units.")
|
||||
|
||||
sc1, sc2, sc3, sc4 = st.columns(4)
|
||||
total_sales_volume = filtered_df["total_eas_cspc"].fillna(0).sum() if "total_eas_cspc" in filtered_df.columns else 0
|
||||
cspc_count = (filtered_df["sales_data_status"] == "VERIFIED_CSPC").sum() if "sales_data_status" in filtered_df.columns else 0
|
||||
riaa_count = (filtered_df["sales_data_status"] == "VERIFIED_RIAA").sum() if "sales_data_status" in filtered_df.columns else 0
|
||||
sea_count = (filtered_df["sales_data_status"] == "ESTIMATED_SEA").sum() if "sales_data_status" in filtered_df.columns else 0
|
||||
|
||||
with sc1:
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title">Tracked Catalog Volume</div><div class="kpi-value">{total_sales_volume/1e6:.1f}M</div><div class="kpi-subtitle">Total Equivalent Album Sales (EAS)</div></div>', unsafe_allow_html=True)
|
||||
with sc2:
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #10b981;">CSPC Verified</div><div class="kpi-value" style="color: #10b981;">{cspc_count}</div><div class="kpi-subtitle">ChartMasters Full Dossiers</div></div>', unsafe_allow_html=True)
|
||||
with sc3:
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #38bdf8;">RIAA Certified</div><div class="kpi-value" style="color: #38bdf8;">{riaa_count}</div><div class="kpi-subtitle">Domestic Certifications</div></div>', unsafe_allow_html=True)
|
||||
with sc4:
|
||||
st.markdown(f'<div class="kpi-box"><div class="kpi-title" style="color: #a855f7;">Streaming SEA</div><div class="kpi-value" style="color: #a855f7;">{sea_count}</div><div class="kpi-subtitle">Luminate/Billboard Conversion</div></div>', unsafe_allow_html=True)
|
||||
|
||||
st.markdown("#### 🏆 All-Time Global Sales Leaderboard")
|
||||
sales_table_df = filtered_df.sort_values(by="total_eas_cspc", ascending=False).copy() if "total_eas_cspc" in filtered_df.columns else filtered_df.copy()
|
||||
if "sales_data_status" in sales_table_df.columns:
|
||||
sales_table_df["provenance_badge"] = sales_table_df["sales_data_status"].apply(get_provenance_badge)
|
||||
|
||||
sales_cols = [
|
||||
"cohort_sales_rank", "chartmasters_sales_rank", "name", "provenance_badge",
|
||||
"total_eas_cspc", "pure_album_sales", "streaming_eas", "physical_singles_sales",
|
||||
"digital_singles_sales", "commercial_power_index", "hiffi_xp_published_rank"
|
||||
]
|
||||
render_s_cols = [c for c in sales_cols if c in sales_table_df.columns]
|
||||
|
||||
st.dataframe(
|
||||
sales_table_df[render_s_cols].reset_index(drop=True),
|
||||
use_container_width=True,
|
||||
height=380,
|
||||
column_config={
|
||||
"cohort_sales_rank": st.column_config.NumberColumn("Cohort Sales Rank", format="#%d", width="small"),
|
||||
"chartmasters_sales_rank": st.column_config.NumberColumn("All-Time Global Rank", format="#%d", width="small"),
|
||||
"name": st.column_config.TextColumn("Artist", width="medium"),
|
||||
"provenance_badge": st.column_config.TextColumn("Audit Provenance", width="medium"),
|
||||
"total_eas_cspc": st.column_config.NumberColumn("Total EAS Units", format="%,d", width="medium"),
|
||||
"pure_album_sales": st.column_config.NumberColumn("Pure Studio Albums", format="%,d", width="medium"),
|
||||
"streaming_eas": st.column_config.NumberColumn("Streaming EAS", format="%,d", width="medium"),
|
||||
"physical_singles_sales": st.column_config.NumberColumn("Physical Singles", format="%,d", width="small"),
|
||||
"digital_singles_sales": st.column_config.NumberColumn("Digital Downloads", format="%,d", width="small"),
|
||||
"commercial_power_index": st.column_config.ProgressColumn("CPI Score", min_value=0, max_value=100, format="%.1f", width="small"),
|
||||
"hiffi_xp_published_rank": st.column_config.NumberColumn("HIFFI Base Rank", format="#%d", width="small"),
|
||||
},
|
||||
)
|
||||
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
c_s1, c_s2 = st.columns(2)
|
||||
with c_s1:
|
||||
if "total_eas_cspc" in sales_table_df.columns:
|
||||
top_sellers = sales_table_df[sales_table_df["total_eas_cspc"].fillna(0) > 0].head(12)
|
||||
fig_format = go.Figure()
|
||||
fig_format.add_trace(go.Bar(name="Traditional Albums", x=top_sellers["name"], y=top_sellers.get("pure_album_sales", 0), marker_color="#8b5cf6"))
|
||||
fig_format.add_trace(go.Bar(name="Digital Singles", x=top_sellers["name"], y=top_sellers.get("digital_singles_sales", 0), marker_color="#38bdf8"))
|
||||
fig_format.add_trace(go.Bar(name="Streaming EAS", x=top_sellers["name"], y=top_sellers.get("streaming_eas", 0), marker_color="#10b981"))
|
||||
fig_format.update_layout(
|
||||
barmode="stack", title="Format Contribution: Pure Albums vs Downloads vs Streaming EAS",
|
||||
template="plotly_dark", paper_bgcolor="#111827", plot_bgcolor="#111827",
|
||||
font=dict(color="#9ca3af"), height=360, margin=dict(l=20, r=20, t=50, b=20),
|
||||
legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1.0),
|
||||
)
|
||||
st.plotly_chart(fig_format, use_container_width=True)
|
||||
|
||||
with c_s2:
|
||||
if "total_eas_cspc" in sales_table_df.columns and "hiffi_xp_score" in sales_table_df.columns:
|
||||
scatter_data = sales_table_df.dropna(subset=["total_eas_cspc", "hiffi_xp_score"]).copy()
|
||||
if not scatter_data.empty:
|
||||
fig_quad = px.scatter(
|
||||
scatter_data, x="hiffi_xp_score", y="total_eas_cspc", size="total_eas_cspc",
|
||||
color="provenance_badge" if "provenance_badge" in scatter_data.columns else None,
|
||||
hover_name="name", log_y=True,
|
||||
title="Legacy vs. Velocity Quadrant (All-Time Sales vs HIFFI-XP)",
|
||||
labels={"hiffi_xp_score": "Current Velocity (HIFFI-XP)", "total_eas_cspc": "Total Lifetime EAS Units (Log Scale)"},
|
||||
template="plotly_dark",
|
||||
)
|
||||
fig_quad.update_layout(paper_bgcolor="#111827", plot_bgcolor="#111827", font=dict(color="#9ca3af"), height=360, margin=dict(l=20, r=20, t=50, b=20))
|
||||
st.plotly_chart(fig_quad, use_container_width=True)
|
||||
|
||||
else:
|
||||
top3 = filtered_df.sort_values(by="hiffi_xp_published_rank", ascending=True).head(3)
|
||||
if len(top3) >= 3 and view_mode != "🧪 Commercial Simulation Engine (CPI)":
|
||||
st.markdown("#### 👑 Market Leaders (Published Baseline Rankings)")
|
||||
if len(top3) >= 3:
|
||||
st.markdown("#### Market Leaders (Audited Multi-Platform Baseline)")
|
||||
p1, p2, p3 = st.columns(3)
|
||||
podium_data = [
|
||||
(p1, top3.iloc[0], "🥇 PUBLISHED #1", "podium-gold", "#f59e0b"),
|
||||
(p2, top3.iloc[1], "🥈 PUBLISHED #2", "podium-silver", "#94a3b8"),
|
||||
(p3, top3.iloc[2], "🥉 PUBLISHED #3", "podium-bronze", "#f97316"),
|
||||
(p1, top3.iloc[0], "PUBLISHED #1", "podium-gold", "#f59e0b"),
|
||||
(p2, top3.iloc[1], "PUBLISHED #2", "podium-silver", "#94a3b8"),
|
||||
(p3, top3.iloc[2], "PUBLISHED #3", "podium-bronze", "#f97316"),
|
||||
]
|
||||
for col, row_data, rank_label, css_class, accent_color in podium_data:
|
||||
tier_val = int(row_data.get("platform_evidence_tier", 1))
|
||||
yt_str = f"{int(row_data['subscriber_count']):,}" if pd.notna(row_data.get("subscriber_count")) else "NaN"
|
||||
sp_str = f"{int(row_data['spotify_monthly_listeners']):,}" if pd.notna(row_data.get("spotify_monthly_listeners")) else "NaN"
|
||||
ig_str = f"{int(row_data['instagram_followers']):,}" if pd.notna(row_data.get("instagram_followers")) else "NaN"
|
||||
with col:
|
||||
st.markdown(
|
||||
f"""
|
||||
<div class="podium-tile {css_class}">
|
||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 6px;">
|
||||
<span style="font-weight: 800; color: {accent_color}; font-size: 0.85rem;">{rank_label}</span>
|
||||
<span class="badge badge-tier{tier_val}">Tier {tier_val} Evidence</span>
|
||||
</div>
|
||||
<div style="font-size: 1.35rem; font-weight: 800; color: #ffffff; margin-bottom: 2px;">{row_data['name']}</div>
|
||||
<div style="font-size: 0.8rem; color: #6b7280; margin-bottom: 12px;">{row_data.get('channel_title', '')}</div>
|
||||
<div style="display: grid; grid-template-columns: repeat(4, 1fr); gap: 6px; background: #0b0f17; padding: 10px 10px; border-radius: 8px;">
|
||||
<div><div style="font-size: 0.64rem; color: #9ca3af;">HIFFI-XP</div><div style="font-size: 1.15rem; font-weight: 800; color: {accent_color};">{row_data.get('hiffi_xp_score', 0.0):.2f}</div></div>
|
||||
<div><div style="font-size: 0.64rem; color: #9ca3af;">YT Subs</div><div style="font-size: 0.95rem; font-weight: 700; color: #ef4444;">{yt_str}</div></div>
|
||||
<div><div style="font-size: 0.64rem; color: #9ca3af;">Spotify</div><div style="font-size: 0.95rem; font-weight: 700; color: #38bdf8;">{sp_str}</div></div>
|
||||
<div><div style="font-size: 0.64rem; color: #9ca3af;">IG Follow</div><div style="font-size: 0.95rem; font-weight: 700; color: #e5e7eb;">{ig_str}</div></div>
|
||||
</div>
|
||||
</div>
|
||||
""",
|
||||
unsafe_allow_html=True,
|
||||
tier_val = int(row_data.get("platform_evidence_tier", 3))
|
||||
sp_num = int(row_data.get("spotify_monthly_listeners") or 0)
|
||||
yt_num = int(row_data.get("subscriber_count") or 0)
|
||||
ig_num = int(row_data.get("instagram_followers") or 0)
|
||||
xp_val = float(row_data.get("hiffi_xp_score") or 0.0)
|
||||
artist_name = str(row_data.get("name", "Unknown"))
|
||||
|
||||
podium_html = (
|
||||
f'<div class="podium-tile {css_class}">'
|
||||
f'<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 8px;">'
|
||||
f'<span style="font-weight: 800; color: {accent_color}; font-size: 0.85rem;">{rank_label}</span>'
|
||||
f'<span class="badge badge-tier{tier_val}">Tier {tier_val} Evidence</span>'
|
||||
f'</div>'
|
||||
f'<div style="font-size: 1.45rem; font-weight: 800; color: #ffffff; margin-bottom: 14px;">{artist_name}</div>'
|
||||
f'<div style="display: grid; grid-template-columns: repeat(4, 1fr); gap: 6px; background: #0b0f17; padding: 10px; border-radius: 8px; text-align: center;">'
|
||||
f'<div><div style="font-size: 0.64rem; color: #9ca3af;">HIFFI-XP</div><div style="font-size: 1.12rem; font-weight: 800; color: {accent_color};">{xp_val:.2f}</div></div>'
|
||||
f'<div><div style="font-size: 0.64rem; color: #9ca3af;">Spotify</div><div style="font-size: 0.90rem; font-weight: 700; color: #38bdf8;">{sp_num:,}</div></div>'
|
||||
f'<div><div style="font-size: 0.64rem; color: #9ca3af;">YT Subs</div><div style="font-size: 0.90rem; font-weight: 700; color: #ef4444;">{yt_num:,}</div></div>'
|
||||
f'<div><div style="font-size: 0.64rem; color: #9ca3af;">IG Follow</div><div style="font-size: 0.90rem; font-weight: 700; color: #e5e7eb;">{ig_num:,}</div></div>'
|
||||
f'</div>'
|
||||
f'</div>'
|
||||
)
|
||||
with col:
|
||||
st.markdown(podium_html, unsafe_allow_html=True)
|
||||
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
if view_mode == "🧪 Commercial Simulation Engine (CPI)":
|
||||
st.markdown("#### 🧪 Simulation Impact & Decision Summary")
|
||||
sm1, sm2, sm3, sm4 = st.columns(4)
|
||||
st.markdown(f"#### {view_mode}")
|
||||
|
||||
g_delta = f"+{sim_metrics.get('top_gainer_delta', 0)}" if sim_metrics.get('top_gainer_delta', 0) > 0 else f"{sim_metrics.get('top_gainer_delta', 0)}"
|
||||
l_delta = f"{sim_metrics.get('top_loser_delta', 0)}"
|
||||
|
||||
with sm1:
|
||||
st.markdown(f'<div class="sim-kpi-card"><div class="sim-kpi-title">Top Gainer</div><div class="sim-kpi-val" style="color: #10b981;">{sim_metrics.get("top_gainer_name", "None")} <span style="font-size: 0.85rem;">(▲ {g_delta})</span></div></div>', unsafe_allow_html=True)
|
||||
with sm2:
|
||||
st.markdown(f'<div class="sim-kpi-card"><div class="sim-kpi-title">Top Loser</div><div class="sim-kpi-val" style="color: #ef4444;">{sim_metrics.get("top_loser_name", "None")} <span style="font-size: 0.85rem;">(▼ {l_delta})</span></div></div>', unsafe_allow_html=True)
|
||||
with sm3:
|
||||
st.markdown(f'<div class="sim-kpi-card"><div class="sim-kpi-title">Mean Absolute Shift</div><div class="sim-kpi-val" style="color: #38bdf8;">± {sim_metrics.get("mean_abs_delta", 0.0)} <span style="font-size: 0.85rem; color: #94a3b8;">positions</span></div></div>', unsafe_allow_html=True)
|
||||
with sm4:
|
||||
st.markdown(f'<div class="sim-kpi-card"><div class="sim-kpi-title">Simulation Scope</div><div class="sim-kpi-val" style="color: #f59e0b;">{sim_metrics.get("matched_coverage", 0)} / {len(filtered_df)} <span style="font-size: 0.85rem; color: #94a3b8;">({sim_mode})</span></div></div>', unsafe_allow_html=True)
|
||||
|
||||
st.caption(f"💡 **Executive Takeaway:** At **{int(sim_weight*100)}% commercial weight**, the average cataloged artist moves **±{sim_metrics.get('mean_abs_delta', 0.0)} positions**.")
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
|
||||
st.markdown(f"#### 📋 {view_mode}")
|
||||
|
||||
if view_mode == "🏆 Published Multi-Platform Rank":
|
||||
if view_mode == "Published Multi-Platform Rank":
|
||||
table_df = filtered_df.sort_values(by="hiffi_xp_published_rank", ascending=True).copy()
|
||||
# Explicitly placing subscriber_count right after HIFFI-XP Score
|
||||
cols = [
|
||||
"hiffi_xp_published_rank", "hiffi_xp_rank", "name",
|
||||
"platform_evidence_tier", "hiffi_xp_score", "subscriber_count",
|
||||
"spotify_monthly_listeners", "instagram_followers",
|
||||
"effective_weights", "anomaly_flags"
|
||||
"hiffi_xp_published_rank", "name", "platform_evidence_tier",
|
||||
"hiffi_xp_score", "spotify_monthly_listeners", "subscriber_count",
|
||||
"instagram_followers", "effective_weights", "anomaly_flags"
|
||||
]
|
||||
elif view_mode == "🚀 Breakout Velocity (BVI)":
|
||||
elif view_mode == "Breakout Velocity (BVI)":
|
||||
bvi_series = filtered_df.get("breakout_velocity_index", pd.Series(0.0, index=filtered_df.index)).fillna(0.0)
|
||||
table_df = filtered_df[bvi_series.notna()].copy()
|
||||
table_df = filtered_df.copy()
|
||||
table_df["breakout_velocity_index"] = bvi_series
|
||||
table_df = table_df.sort_values(by="breakout_velocity_index", ascending=False)
|
||||
cols = ["name", "breakout_velocity_index", "hiffi_xp_published_rank", "daily_view_velocity", "momentum_30d", "subscriber_count", "subgenre"]
|
||||
elif view_mode == "🧪 Commercial Simulation Engine (CPI)":
|
||||
table_df = filtered_df.sort_values(by="simulated_rank", ascending=True).copy() if "simulated_rank" in filtered_df.columns else filtered_df.copy()
|
||||
|
||||
def format_rank_delta(val):
|
||||
if pd.isna(val):
|
||||
return "N/A"
|
||||
ival = int(val)
|
||||
if ival > 0:
|
||||
return f"▲ +{ival} (Up)"
|
||||
elif ival < 0:
|
||||
return f"▼ {ival} (Down)"
|
||||
else:
|
||||
return "— 0 (Equal)"
|
||||
|
||||
if "simulation_rank_delta" in table_df.columns:
|
||||
table_df["rank_shift_indicator"] = table_df["simulation_rank_delta"].apply(format_rank_delta)
|
||||
else:
|
||||
table_df["rank_shift_indicator"] = "— 0"
|
||||
|
||||
comp_rank = "baseline_matched_rank" if sim_mode == "matched_only" else "hiffi_xp_published_rank"
|
||||
|
||||
cols = [
|
||||
"simulated_rank", comp_rank, "rank_shift_indicator", "name",
|
||||
"simulated_hiffi_score", "hiffi_xp_score", "commercial_power_index",
|
||||
"total_eas_cspc", "sales_data_status",
|
||||
]
|
||||
else:
|
||||
table_df = filtered_df.sort_values(by="hiffi_xp_published_rank", ascending=True).copy()
|
||||
cols = ["hiffi_xp_published_rank", "name", "production_hiffi_rank", "hiffi_xp_rank", "rank_delta", "platform_evidence_tier", "hiffi_xp_score", "subscriber_count", "spotify_monthly_listeners", "instagram_followers", "total_eas_cspc", "sales_data_status", "evidence_exclusions", "anomaly_flags"]
|
||||
cols = ["hiffi_xp_published_rank", "name", "platform_evidence_tier", "hiffi_xp_score", "spotify_monthly_listeners", "subscriber_count", "instagram_followers", "daily_view_velocity", "anomaly_flags"]
|
||||
|
||||
display_table = table_df.copy()
|
||||
if "sales_data_status" in display_table.columns:
|
||||
display_table["sales_data_status"] = display_table["sales_data_status"].apply(get_provenance_badge)
|
||||
|
||||
render_cols = [c for c in cols if c in display_table.columns]
|
||||
|
||||
st.dataframe(
|
||||
display_table[render_cols].reset_index(drop=True),
|
||||
use_container_width=True,
|
||||
height=340,
|
||||
width="stretch",
|
||||
height=420,
|
||||
column_config={
|
||||
"hiffi_xp_published_rank": st.column_config.NumberColumn("Published Rank", format="%d", width="small"),
|
||||
"hiffi_xp_rank": st.column_config.NumberColumn("Raw Rank", format="%d", width="small"),
|
||||
"hiffi_xp_published_rank": st.column_config.NumberColumn("Rank", format="#%d", width="small"),
|
||||
"name": st.column_config.TextColumn("Artist", width="medium"),
|
||||
"platform_evidence_tier": st.column_config.NumberColumn("Tier", format="Tier %d", width="small"),
|
||||
"baseline_matched_rank": st.column_config.NumberColumn("Baseline Match Rank", format="%d", width="small"),
|
||||
"simulated_rank": st.column_config.NumberColumn("Simulated Rank", format="%d", width="small"),
|
||||
"rank_shift_indicator": st.column_config.TextColumn("Simulated Δ (Movement)", width="medium"),
|
||||
"simulated_hiffi_score": st.column_config.ProgressColumn("Simulated Score", format="%.2f", min_value=0, max_value=100, width="medium"),
|
||||
"hiffi_xp_score": st.column_config.ProgressColumn("HIFFI-XP Score", format="%.2f", min_value=0, max_value=100, width="medium"),
|
||||
"platform_evidence_tier": st.column_config.NumberColumn("Evidence Tier", format="Tier %d", width="small"),
|
||||
"hiffi_xp_score": st.column_config.ProgressColumn("HIFFI-XP Score", min_value=0, max_value=100, format="%.2f", width="medium"),
|
||||
"spotify_monthly_listeners": st.column_config.NumberColumn("Spotify Listeners", format="%,d", width="medium"),
|
||||
"subscriber_count": st.column_config.NumberColumn("YouTube Subscribers", format="%,d", width="medium"),
|
||||
"spotify_monthly_listeners": st.column_config.NumberColumn("Spotify Monthly Listeners", format="%,d", width="medium"),
|
||||
"instagram_followers": st.column_config.NumberColumn("Instagram Followers", format="%,d", width="medium"),
|
||||
"effective_weights": st.column_config.TextColumn("Weights Distribution", width="medium"),
|
||||
"anomaly_flags": st.column_config.TextColumn("Integrity Audit", width="small"),
|
||||
"breakout_velocity_index": st.column_config.ProgressColumn("Breakout Velocity Index", format="%.2f", min_value=0, max_value=100, width="medium"),
|
||||
"commercial_power_index": st.column_config.ProgressColumn("Commercial Power Index", format="%.1f", min_value=0, max_value=100, width="medium"),
|
||||
"total_eas_cspc": st.column_config.NumberColumn("Total Equivalent Album Sales", format="%,d", width="medium"),
|
||||
"pure_album_sales": st.column_config.NumberColumn("Traditional Album Sales", format="%,d", width="medium"),
|
||||
"streaming_eas": st.column_config.NumberColumn("Streaming-Equivalent Album Sales", format="%,d", width="medium"),
|
||||
"sales_data_status": st.column_config.TextColumn("Sales Audit Status", width="medium"),
|
||||
"daily_view_velocity": st.column_config.NumberColumn("Daily Video Velocity", format="%,d", width="small"),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -803,60 +488,57 @@ else:
|
||||
chart_df, x="spotify_monthly_listeners", y="instagram_followers",
|
||||
size="hiffi_xp_score", color="platform_evidence_tier", hover_name="name",
|
||||
log_x=True, log_y=True, title="Spotify Monthly Listeners vs. Instagram Followers",
|
||||
labels={"spotify_monthly_listeners": "Spotify Monthly Listeners (Log Scale)", "instagram_followers": "Instagram Followers (Log Scale)", "platform_evidence_tier": "Tier"},
|
||||
labels={"spotify_monthly_listeners": "Spotify Listeners (Log)", "instagram_followers": "Instagram Followers (Log)", "platform_evidence_tier": "Tier"},
|
||||
template="plotly_dark",
|
||||
)
|
||||
fig_scatter.update_layout(paper_bgcolor="#111827", plot_bgcolor="#111827", font=dict(color="#9ca3af"), height=320, margin=dict(l=20, r=20, t=40, b=20))
|
||||
st.plotly_chart(fig_scatter, use_container_width=True)
|
||||
st.plotly_chart(fig_scatter, width="stretch")
|
||||
with c_right:
|
||||
if "platform_evidence_tier" in filtered_df.columns:
|
||||
tier_counts = filtered_df["platform_evidence_tier"].value_counts().reset_index()
|
||||
tier_counts.columns = ["Tier", "Count"]
|
||||
fig_pie = px.pie(tier_counts, values="Count", names="Tier", title="Evidence Tier Cohort Distribution", template="plotly_dark", color="Tier", color_discrete_map={3: "#10b981", 2: "#38bdf8", 1: "#f59e0b", 0: "#ef4444"})
|
||||
fig_pie.update_layout(paper_bgcolor="#111827", plot_bgcolor="#111827", font=dict(color="#9ca3af"), height=320, margin=dict(l=20, r=20, t=40, b=20))
|
||||
st.plotly_chart(fig_pie, use_container_width=True)
|
||||
st.plotly_chart(fig_pie, width="stretch")
|
||||
|
||||
# Dossier for 500
|
||||
st.markdown("<div class='custom-divider'></div>", unsafe_allow_html=True)
|
||||
st.markdown("#### 🔎 Artist Dossier")
|
||||
st.markdown("#### Artist Dossier")
|
||||
if "name" in df.columns and len(df["name"].unique()) > 0:
|
||||
selected_name = st.selectbox("Inspect Artist Details:", df["name"].unique(), label_visibility="collapsed")
|
||||
row = df[df["name"] == selected_name].iloc[0]
|
||||
matched_rows = df[df["name"] == selected_name]
|
||||
if not matched_rows.empty:
|
||||
row = matched_rows.iloc[0]
|
||||
|
||||
d1, d2, d3, d4, d5, d6 = st.columns(6)
|
||||
d1, d2, d3, d4, d5 = st.columns(5)
|
||||
with d1:
|
||||
h_val = row.get("hiffi_xp_score")
|
||||
st.metric("HIFFI-XP Score", f"{h_val:.2f}" if pd.notna(h_val) else "N/A")
|
||||
st.metric("HIFFI-XP Score", f"{float(row.get('hiffi_xp_score') or 0.0):.2f}")
|
||||
with d2:
|
||||
pub_rank = row.get("hiffi_xp_published_rank")
|
||||
st.metric("Published Rank", f"#{int(pub_rank)}" if pd.notna(pub_rank) else "Unranked")
|
||||
with d3:
|
||||
yt_s = row.get("subscriber_count")
|
||||
st.metric("YouTube Subscribers", f"{int(yt_s):,}" if pd.notna(yt_s) else "Unindexed (NaN)")
|
||||
with d4:
|
||||
sp_l = row.get("spotify_monthly_listeners")
|
||||
st.metric("Spotify Listeners", f"{int(sp_l):,}" if pd.notna(sp_l) else "Unindexed (NaN)")
|
||||
st.metric("Spotify Listeners", f"{int(sp_l):,}" if pd.notna(sp_l) else "0")
|
||||
with d4:
|
||||
yt_s = row.get("subscriber_count")
|
||||
st.metric("YouTube Subscribers", f"{int(yt_s):,}" if pd.notna(yt_s) else "0")
|
||||
with d5:
|
||||
ig_f = row.get("instagram_followers")
|
||||
st.metric("Instagram Followers", f"{int(ig_f):,}" if pd.notna(ig_f) else "Unindexed (NaN)")
|
||||
with d6:
|
||||
sales_val = row.get("total_eas_cspc")
|
||||
st.metric("Total EAS Sales", f"{int(sales_val):,}" if pd.notna(sales_val) else "Awaiting Ingestion")
|
||||
st.metric("Instagram Followers", f"{int(ig_f):,}" if pd.notna(ig_f) else "0")
|
||||
|
||||
st.markdown(
|
||||
f"""
|
||||
<div style="background: #111827; padding: 14px 18px; border-radius: 8px; border: 1px solid #1f2937; margin-top: 10px; font-size: 0.85rem;">
|
||||
<div style="display: flex; gap: 20px; flex-wrap: wrap;">
|
||||
<div><b>Artist:</b> {row['name']}</div>
|
||||
<div><b>Evidence Tier:</b> Tier {int(row.get('platform_evidence_tier', 1))}</div>
|
||||
<div><b>YouTube Subscribers:</b> {f"{int(row.get('subscriber_count')):,}" if pd.notna(row.get('subscriber_count')) else "N/A"}</div>
|
||||
<div><b>Global Sales Rank:</b> {f"#{int(row.get('chartmasters_sales_rank'))}" if pd.notna(row.get('chartmasters_sales_rank')) else "Unranked"}</div>
|
||||
<div><b>Cohort Sales Rank:</b> {f"#{int(row.get('cohort_sales_rank'))}" if pd.notna(row.get('cohort_sales_rank')) else "Uncataloged"}</div>
|
||||
<div><b>Commercial Power Index:</b> {f"{row.get('commercial_power_index'):.1f}" if pd.notna(row.get('commercial_power_index')) else "Awaiting Audit"}</div>
|
||||
<div><b>Traditional Albums:</b> {f"{int(row.get('pure_album_sales')):,}" if pd.notna(row.get('pure_album_sales')) else "N/A"}</div>
|
||||
<div><b>Streaming-Equivalent EAS:</b> {f"{int(row.get('streaming_eas')):,}" if pd.notna(row.get('streaming_eas')) else "N/A"}</div>
|
||||
<div><b>Sales Status:</b> {get_provenance_badge(row.get('sales_data_status', ''))}</div>
|
||||
</div>
|
||||
</div>
|
||||
""",
|
||||
unsafe_allow_html=True,
|
||||
artist_disp_name = str(row["name"])
|
||||
tier_disp_num = int(row.get("platform_evidence_tier", 1))
|
||||
bvi_val = float(row.get("breakout_velocity_index") or 0.0)
|
||||
weights_disp = str(row.get("effective_weights", "sp:45.0%,yt:35.0%,ig:20.0%"))
|
||||
|
||||
dossier_html = (
|
||||
f'<div style="background: #111827; padding: 14px 18px; border-radius: 8px; border: 1px solid #1f2937; margin-top: 10px; font-size: 0.85rem;">'
|
||||
f'<div style="display: flex; gap: 24px; flex-wrap: wrap;">'
|
||||
f'<div><b>Artist:</b> {artist_disp_name}</div>'
|
||||
f'<div><b>Evidence Tier:</b> Tier {tier_disp_num}</div>'
|
||||
f'<div><b>Breakout Velocity Index:</b> {bvi_val:.2f}</div>'
|
||||
f'<div><b>Telemetry Weighting:</b> {weights_disp}</div>'
|
||||
f'<div><b>Integrity Audit:</b> {row.get("anomaly_flags", "CLEAN")}</div>'
|
||||
f'</div>'
|
||||
f'</div>'
|
||||
)
|
||||
st.markdown(dossier_html, unsafe_allow_html=True)
|
||||
+201
-401
@@ -1,457 +1,257 @@
|
||||
"""
|
||||
core/ranking_engine.py
|
||||
Ranks artists according to HIFFI formula stages:
|
||||
- YouTube Production Baseline: Momentum (35%), Velocity (30%), Reach (20%), Consistency (15%)
|
||||
- Breakout Velocity Index (BVI, 0-100) with 10k subscriber floor
|
||||
- Multi-Platform HIFFI-XP Composite (0-100):
|
||||
* YouTube Production Score (50%)
|
||||
* Spotify Monthly Listeners (25%)
|
||||
* Spotify Daily Change / Velocity (5%)
|
||||
* Instagram Followers (20%)
|
||||
With zero-imputation compliance, proportional weight redistribution,
|
||||
and effective weight serialization.
|
||||
- Publication Eligibility Gates: Decoupled public chart assigning consecutive ranks
|
||||
while enforcing multi-platform evidence tiers (Top 10 = Tier 3, Top 50 = Tier >= 2)
|
||||
conditioned on strict identity verification, status_flag validation, and UTC data freshness thresholds.
|
||||
Fail-closed on missing timestamp columns, failed lookups, or unverified statuses.
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple, Union
|
||||
import math
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Optional, Tuple, Union
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class HIFFIRankingEngine:
|
||||
|
||||
@staticmethod
|
||||
def _log_scale(series: pd.Series) -> pd.Series:
|
||||
valid = series.dropna()
|
||||
log_vals = np.log1p(np.maximum(valid.astype(float), 0.0))
|
||||
s_min, s_max = log_vals.min(), log_vals.max()
|
||||
if np.isclose(s_max, s_min) or pd.isna(s_min):
|
||||
return pd.Series(0.0, index=series.index)
|
||||
scaled = (log_vals - s_min) / (s_max - s_min + 1e-9)
|
||||
return scaled.reindex(series.index)
|
||||
|
||||
@staticmethod
|
||||
def _min_max_scale(series: pd.Series) -> pd.Series:
|
||||
valid = series.dropna()
|
||||
s_min, s_max = valid.min(), valid.max()
|
||||
if np.isclose(s_max, s_min) or pd.isna(s_min):
|
||||
return pd.Series(0.0, index=series.index)
|
||||
scaled = (valid - s_min) / (s_max - s_min + 1e-9)
|
||||
return scaled.reindex(series.index)
|
||||
|
||||
@staticmethod
|
||||
def _is_signal_fresh(
|
||||
timestamp_val: object,
|
||||
max_days: float,
|
||||
reference_time: Optional[pd.Timestamp] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Parses UTC ISO 8601 string or pd.Timestamp and checks staleness.
|
||||
Fails closed (returns False) on missing, null, or unparseable timestamps.
|
||||
HIFFI-XP Core Ranking Engine & Governance Suite.
|
||||
- 45/35/20 Bounded log-scaled scoring with dynamic weight redistribution.
|
||||
- Publication eligibility gates (timestamp freshness, identity verification, status flags).
|
||||
- Statistical anomaly detection for ratio spikes and unnatural patterns.
|
||||
"""
|
||||
if pd.isna(timestamp_val):
|
||||
return False
|
||||
|
||||
try:
|
||||
ts = pd.to_datetime(timestamp_val, utc=True)
|
||||
ref = reference_time or pd.Timestamp.now(tz="UTC")
|
||||
age_days = (ref - ts).total_seconds() / 86400.0
|
||||
return 0.0 <= age_days <= max_days
|
||||
except Exception:
|
||||
return False
|
||||
def __init__(self):
|
||||
self.MAX_SPOTIFY = 120_000_000.0
|
||||
self.MAX_YT_SUBS = 70_000_000.0
|
||||
self.MAX_YT_VELOCITY = 5_000_000.0
|
||||
self.MAX_INSTAGRAM = 150_000_000.0
|
||||
|
||||
self.w_sp = 0.45
|
||||
self.w_yt = 0.35
|
||||
self.w_ig = 0.20
|
||||
|
||||
def _log_scale(self, val: float, max_bound: float) -> float:
|
||||
if pd.isna(val) or val <= 0:
|
||||
return 0.0
|
||||
s = math.log10(float(val) + 1.0)
|
||||
mx = math.log10(float(max_bound) + 1.0)
|
||||
return float(np.clip((s / mx) * 100.0, 0.0, 100.0))
|
||||
|
||||
def calculate_breakout_velocity(self, df: pd.DataFrame) -> pd.Series:
|
||||
"""
|
||||
Computes the Breakout Velocity Index (BVI, 0 - 100).
|
||||
Requires at least 10,000 subscribers and positive velocity to qualify.
|
||||
"""
|
||||
out_bvi = pd.Series(np.nan, index=df.index, dtype=float)
|
||||
|
||||
status_col = df["status_flag"] if "status_flag" in df.columns else pd.Series("COMPLETE", index=df.index)
|
||||
|
||||
valid_mask = (
|
||||
df["subscriber_count"].notna()
|
||||
& (df["subscriber_count"] >= 10000)
|
||||
& df["daily_view_velocity"].notna()
|
||||
& (df["daily_view_velocity"] > 0.0)
|
||||
& (status_col != "INACTIVE_OR_DORMANT")
|
||||
& (status_col != "FAILED_LOOKUP")
|
||||
)
|
||||
|
||||
if not valid_mask.any():
|
||||
return out_bvi
|
||||
|
||||
subset = df[valid_mask]
|
||||
subs = subset["subscriber_count"].astype(float)
|
||||
daily_vel = subset["daily_view_velocity"].astype(float)
|
||||
momentum = subset["momentum_30d"].fillna(0.0).clip(lower=-100.0, upper=300.0)
|
||||
|
||||
v_aud = daily_vel / np.maximum(subs, 1000.0)
|
||||
m_signal = (momentum + 100.0) / 400.0
|
||||
log_v_aud = np.log10(1.0 + v_aud)
|
||||
w_emerge = 1.0 / (1.0 + np.log10(1.0 + np.maximum(subs, 1.0)))
|
||||
|
||||
raw_bvi = (0.65 * log_v_aud + 0.35 * m_signal) * w_emerge
|
||||
|
||||
bvi_min = raw_bvi.min()
|
||||
bvi_max = raw_bvi.max()
|
||||
|
||||
if np.isclose(bvi_max, bvi_min):
|
||||
scaled_bvi = pd.Series(50.0, index=subset.index)
|
||||
else:
|
||||
scaled_bvi = ((raw_bvi - bvi_min) / (bvi_max - bvi_min + 1e-9)) * 100.0
|
||||
|
||||
out_bvi.loc[valid_mask] = scaled_bvi.round(2)
|
||||
return out_bvi
|
||||
|
||||
def score_and_rank(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
min_confidence: float = 0.50,
|
||||
) -> pd.DataFrame:
|
||||
"""Computes baseline YouTube production ranking."""
|
||||
out = df.copy()
|
||||
|
||||
if "confidence_score" in out.columns:
|
||||
out["confidence_score"] = pd.to_numeric(
|
||||
out["confidence_score"].apply(
|
||||
lambda x: x.get("composite_confidence", 0.0) if isinstance(x, dict) else x
|
||||
),
|
||||
errors="coerce"
|
||||
).fillna(0.0)
|
||||
|
||||
out["breakout_velocity_index"] = self.calculate_breakout_velocity(out)
|
||||
|
||||
status_col = out["status_flag"] if "status_flag" in out.columns else pd.Series("COMPLETE", index=out.index)
|
||||
|
||||
has_baseline = (
|
||||
out["subscriber_count"].notna()
|
||||
& out["lifetime_views"].notna()
|
||||
& (status_col != "FAILED_LOOKUP")
|
||||
)
|
||||
|
||||
out["is_rank_eligible"] = (
|
||||
has_baseline
|
||||
& (out.get("confidence_score", pd.Series(1.0, index=out.index)) >= min_confidence)
|
||||
& (status_col.isin(["COMPLETE", "CATALOG_ACTIVE_NO_RECENT_UPLOAD"]))
|
||||
)
|
||||
|
||||
filled_momentum = out["momentum_30d"].fillna(0.0).clip(lower=-100.0, upper=300.0) if "momentum_30d" in out.columns else pd.Series(0.0, index=out.index)
|
||||
score_momentum = self._min_max_scale(filled_momentum)
|
||||
|
||||
filled_velocity = out["daily_view_velocity"].fillna(0.0) if "daily_view_velocity" in out.columns else pd.Series(0.0, index=out.index)
|
||||
score_velocity = self._log_scale(filled_velocity)
|
||||
|
||||
log_subs = self._log_scale(out["subscriber_count"].fillna(0.0)) if "subscriber_count" in out.columns else pd.Series(0.0, index=out.index)
|
||||
log_lifetime = self._log_scale(out["lifetime_views"].fillna(0.0)) if "lifetime_views" in out.columns else pd.Series(0.0, index=out.index)
|
||||
score_reach = 0.60 * log_subs + 0.40 * log_lifetime
|
||||
|
||||
score_consistency = self._log_scale(out["video_count"].fillna(0.0)) if "video_count" in out.columns else pd.Series(0.0, index=out.index)
|
||||
|
||||
hiffi_raw = (
|
||||
0.35 * score_momentum
|
||||
+ 0.30 * score_velocity
|
||||
+ 0.20 * score_reach
|
||||
+ 0.15 * score_consistency
|
||||
) * 100.0
|
||||
|
||||
conf_multiplier = 0.8 + 0.2 * out.get("confidence_score", pd.Series(1.0, index=out.index)).clip(0.0, 1.0)
|
||||
out["production_hiffi_score"] = (hiffi_raw * conf_multiplier).round(2)
|
||||
|
||||
eligible_mask = out["is_rank_eligible"]
|
||||
out["production_hiffi_rank"] = pd.NA
|
||||
out.loc[eligible_mask, "production_hiffi_rank"] = (
|
||||
out.loc[eligible_mask, "production_hiffi_score"]
|
||||
.rank(ascending=False, method="min")
|
||||
.astype(int)
|
||||
)
|
||||
|
||||
bvi_eligible_mask = eligible_mask & out["breakout_velocity_index"].notna()
|
||||
out["breakout_rank"] = pd.NA
|
||||
out.loc[bvi_eligible_mask, "breakout_rank"] = (
|
||||
out.loc[bvi_eligible_mask, "breakout_velocity_index"]
|
||||
.rank(ascending=False, method="min")
|
||||
.astype(int)
|
||||
)
|
||||
|
||||
return out
|
||||
if df.empty:
|
||||
return pd.Series(dtype=float)
|
||||
growth_7d = pd.to_numeric(df.get("growth_velocity_7d", 0.0), errors="coerce").fillna(0.0)
|
||||
growth_30d = pd.to_numeric(df.get("growth_velocity_30d", 0.0), errors="coerce").fillna(0.0)
|
||||
raw_bvi = (growth_7d * 0.6 + growth_30d * 0.4) * 10.0
|
||||
return pd.Series(np.clip(raw_bvi, 0.0, 100.0), index=df.index).round(2)
|
||||
|
||||
def calculate_hiffi_xp(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Calculates HIFFI-XP composite multi-platform score, raw rank, and effective weights.
|
||||
Enforces proportional weight redistribution when signals are missing (NaN).
|
||||
Weights: YouTube (0.50), Spotify Monthly (0.25), Spotify Velocity (0.05), Instagram (0.20).
|
||||
"""
|
||||
out = df.copy()
|
||||
|
||||
# Schema aliasing for raw input fallbacks
|
||||
if "production_hiffi_score" not in out.columns and "hiffi_score" in out.columns:
|
||||
out["production_hiffi_score"] = out["hiffi_score"]
|
||||
if "production_hiffi_rank" not in out.columns and "hiffi_rank" in out.columns:
|
||||
out["production_hiffi_rank"] = out["hiffi_rank"]
|
||||
has_sp = out.get("spotify_monthly_listeners", pd.Series(0, index=out.index)).fillna(0).gt(0).astype(int)
|
||||
has_yt = out.get("subscriber_count", pd.Series(0, index=out.index)).fillna(0).gt(0).astype(int)
|
||||
has_ig = out.get("instagram_followers", pd.Series(0, index=out.index)).fillna(0).gt(0).astype(int)
|
||||
|
||||
base_weights = {
|
||||
"yt": 0.50,
|
||||
"sp_monthly": 0.25,
|
||||
"sp_velocity": 0.05,
|
||||
"ig": 0.20
|
||||
}
|
||||
if "subscriber_count" not in out.columns:
|
||||
has_yt = out.get("lifetime_views", pd.Series(0, index=out.index)).fillna(0).gt(0).astype(int)
|
||||
if has_yt.sum() == 0 and "status_flag" in out.columns:
|
||||
has_yt = (out["status_flag"] == "COMPLETE").astype(int)
|
||||
|
||||
def log_scale_vector(series: pd.Series) -> pd.Series:
|
||||
valid = series.dropna()
|
||||
if valid.empty:
|
||||
return pd.Series(np.nan, index=series.index)
|
||||
log_val = np.log1p(np.maximum(0.0, valid.astype(float)))
|
||||
min_v, max_v = log_val.min(), log_val.max()
|
||||
if np.isclose(max_v, min_v):
|
||||
return pd.Series(1.0, index=valid.index)
|
||||
scaled = (log_val - min_v) / (max_v - min_v + 1e-9)
|
||||
return scaled.reindex(series.index)
|
||||
out["platform_evidence_tier"] = has_sp + has_yt + has_ig
|
||||
|
||||
def linear_scale_vector(series: pd.Series) -> pd.Series:
|
||||
valid = series.dropna()
|
||||
if valid.empty:
|
||||
return pd.Series(np.nan, index=series.index)
|
||||
min_v, max_v = valid.min(), valid.max()
|
||||
if np.isclose(max_v, min_v):
|
||||
return pd.Series(0.5, index=valid.index)
|
||||
scaled = (valid - min_v) / (max_v - min_v + 1e-9)
|
||||
return scaled.reindex(series.index)
|
||||
confidence_map = {3: 1.00, 2: 0.85, 1: 0.65, 0: 0.50}
|
||||
out["confidence_score"] = out["platform_evidence_tier"].map(confidence_map).fillna(1.0)
|
||||
|
||||
s_yt = log_scale_vector(out["production_hiffi_score"]) if "production_hiffi_score" in out.columns else pd.Series(np.nan, index=out.index)
|
||||
s_sp_m = log_scale_vector(out["spotify_monthly_listeners"]) if "spotify_monthly_listeners" in out.columns else pd.Series(np.nan, index=out.index)
|
||||
s_sp_v = linear_scale_vector(out["spotify_daily_change"]) if "spotify_daily_change" in out.columns else pd.Series(np.nan, index=out.index)
|
||||
s_ig = log_scale_vector(out["instagram_followers"]) if "instagram_followers" in out.columns else pd.Series(np.nan, index=out.index)
|
||||
scores = []
|
||||
effective_weight_strings = []
|
||||
|
||||
signals = pd.DataFrame({
|
||||
"yt": s_yt,
|
||||
"sp_monthly": s_sp_m,
|
||||
"sp_velocity": s_sp_v,
|
||||
"ig": s_ig
|
||||
}, index=out.index)
|
||||
for _, row in out.iterrows():
|
||||
sp_val = row.get("spotify_monthly_listeners")
|
||||
sp_clean = float(sp_val) if pd.notna(sp_val) and float(sp_val) > 0 else 0.0
|
||||
|
||||
# Proportional Dynamic Weight Redistribution Matrix
|
||||
weight_mask = signals.notna()
|
||||
allocated_weights = weight_mask.multiply(pd.Series(base_weights), axis=1)
|
||||
weight_sum = allocated_weights.sum(axis=1)
|
||||
yt_sub = row.get("subscriber_count")
|
||||
yt_sub_clean = float(yt_sub) if pd.notna(yt_sub) and float(yt_sub) > 0 else 0.0
|
||||
|
||||
weight_sum = weight_sum.replace(0.0, np.nan)
|
||||
normalized_weights = allocated_weights.divide(weight_sum, axis=0)
|
||||
yt_vel = row.get("daily_view_velocity")
|
||||
yt_vel_clean = float(yt_vel) if pd.notna(yt_vel) and float(yt_vel) > 0 else 0.0
|
||||
|
||||
# Explainability: Format explicit percentage breakdown per artist
|
||||
def format_weights(row: pd.Series) -> str:
|
||||
parts = [f"{col}:{val*100.0:.1f}%" for col, val in row.items() if val > 0]
|
||||
return ",".join(parts) if parts else "NONE"
|
||||
ig_val = row.get("instagram_followers")
|
||||
ig_clean = float(ig_val) if pd.notna(ig_val) and float(ig_val) > 0 else 0.0
|
||||
|
||||
out["effective_weights"] = normalized_weights.apply(format_weights, axis=1)
|
||||
s_sp = self._log_scale(sp_clean, self.MAX_SPOTIFY)
|
||||
s_yt = self._log_scale(yt_sub_clean, self.MAX_YT_SUBS) * 0.70 + self._log_scale(yt_vel_clean, self.MAX_YT_VELOCITY) * 0.30
|
||||
s_ig = self._log_scale(ig_clean, self.MAX_INSTAGRAM)
|
||||
|
||||
component_scores = (signals.fillna(0.0) * normalized_weights).sum(axis=1) * 100.0
|
||||
out["hiffi_xp_score"] = component_scores.round(2)
|
||||
out.loc[weight_sum.isna(), "hiffi_xp_score"] = np.nan
|
||||
active_weights = {}
|
||||
if sp_clean > 0:
|
||||
active_weights["sp"] = self.w_sp
|
||||
if yt_sub_clean > 0 or yt_vel_clean > 0:
|
||||
active_weights["yt"] = self.w_yt
|
||||
if ig_clean > 0:
|
||||
active_weights["ig"] = self.w_ig
|
||||
|
||||
# Deterministic tie-breaking for raw continuous ranking
|
||||
sort_cols = ["hiffi_xp_score", "production_hiffi_score", "spotify_monthly_listeners"]
|
||||
present_sort = [c for c in sort_cols if c in out.columns]
|
||||
total_active = sum(active_weights.values())
|
||||
|
||||
valid_mask = out["hiffi_xp_score"].notna()
|
||||
sorted_indices = (
|
||||
out[valid_mask]
|
||||
.sort_values(by=present_sort, ascending=[False] * len(present_sort))
|
||||
.index
|
||||
)
|
||||
if total_active > 0:
|
||||
norm_sp = active_weights.get("sp", 0.0) / total_active
|
||||
norm_yt = active_weights.get("yt", 0.0) / total_active
|
||||
norm_ig = active_weights.get("ig", 0.0) / total_active
|
||||
|
||||
out["hiffi_xp_rank"] = pd.NA
|
||||
out.loc[sorted_indices, "hiffi_xp_rank"] = range(1, len(sorted_indices) + 1)
|
||||
out.loc[valid_mask, "hiffi_xp_rank"] = out.loc[valid_mask, "hiffi_xp_rank"].astype(int)
|
||||
|
||||
if "production_hiffi_rank" in out.columns:
|
||||
out["rank_delta"] = pd.to_numeric(out["production_hiffi_rank"], errors="coerce") - pd.to_numeric(out["hiffi_xp_rank"], errors="coerce")
|
||||
raw_score = (s_sp * norm_sp) + (s_yt * norm_yt) + (s_ig * norm_ig)
|
||||
conf = confidence_map.get(row.get("platform_evidence_tier", 3), 1.0)
|
||||
final_score = float(round(raw_score * conf, 2))
|
||||
wt_str = f"sp:{round(norm_sp*100,1)}%,yt:{round(norm_yt*100,1)}%,ig:{round(norm_ig*100,1)}%"
|
||||
else:
|
||||
out["rank_delta"] = np.nan
|
||||
prod_score = row.get("production_hiffi_score")
|
||||
if pd.notna(prod_score):
|
||||
final_score = float(round(float(prod_score), 2))
|
||||
else:
|
||||
final_score = 0.0
|
||||
wt_str = "unindexed"
|
||||
|
||||
scores.append(final_score)
|
||||
effective_weight_strings.append(wt_str)
|
||||
|
||||
out["hiffi_xp_score"] = scores
|
||||
out["effective_weights"] = effective_weight_strings
|
||||
|
||||
if "catalog_rank" not in out.columns or out["catalog_rank"].isna().any():
|
||||
out["catalog_rank"] = out["hiffi_xp_score"].rank(ascending=False, method="min").astype(int)
|
||||
out["hiffi_xp_published_rank"] = out["catalog_rank"]
|
||||
|
||||
return out
|
||||
|
||||
def detect_metric_anomalies(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Flags suspicious metric patterns without altering raw scores:
|
||||
- SUSPICIOUS_IG_TO_YT_RATIO: IG followers > 10x YouTube lifetime views.
|
||||
- EXTREME_SPOTIFY_VELOCITY_RATIO: Spotify daily change > 25% of monthly listeners.
|
||||
- Populates 'anomaly_flags' column (semicolon-delimited or 'CLEAN').
|
||||
"""
|
||||
out = df.copy()
|
||||
flags = [[] for _ in range(len(out))]
|
||||
|
||||
# 1. Instagram to YouTube Lifetime Ratio Check
|
||||
if "instagram_followers" in out.columns and "lifetime_views" in out.columns:
|
||||
ig = pd.to_numeric(out["instagram_followers"], errors="coerce")
|
||||
yt_views = pd.to_numeric(out["lifetime_views"], errors="coerce")
|
||||
suspicious_ratio = (ig.notna() & yt_views.notna() & (yt_views > 0) & (ig > 10.0 * yt_views))
|
||||
for idx, is_suspicious in enumerate(suspicious_ratio):
|
||||
if is_suspicious:
|
||||
flags[idx].append("SUSPICIOUS_IG_TO_YT_RATIO")
|
||||
|
||||
# 2. Spotify Velocity to Monthly Listeners Check
|
||||
if "spotify_daily_change" in out.columns and "spotify_monthly_listeners" in out.columns:
|
||||
sp_change = pd.to_numeric(out["spotify_daily_change"], errors="coerce")
|
||||
sp_monthly = pd.to_numeric(out["spotify_monthly_listeners"], errors="coerce")
|
||||
extreme_velocity = (
|
||||
sp_change.notna()
|
||||
& sp_monthly.notna()
|
||||
& (sp_monthly > 1000)
|
||||
& (sp_change.abs() > 0.25 * sp_monthly)
|
||||
)
|
||||
for idx, is_extreme in enumerate(extreme_velocity):
|
||||
if is_extreme:
|
||||
flags[idx].append("EXTREME_SPOTIFY_VELOCITY_RATIO")
|
||||
|
||||
out["anomaly_flags"] = [";".join(f) if f else "CLEAN" for f in flags]
|
||||
return out
|
||||
def score_and_rank(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
return self.calculate_hiffi_xp(df)
|
||||
|
||||
def apply_publication_eligibility_gates(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
reference_time: Optional[pd.Timestamp] = None,
|
||||
reference_time: Optional[Any] = None,
|
||||
return_metadata: bool = False
|
||||
) -> Union[pd.DataFrame, Tuple[pd.DataFrame, dict]]:
|
||||
"""
|
||||
Constructs a publication ranking layer decoupled from raw scores:
|
||||
- Top 10 requires all 3 platforms (Tier 3), strictly verified and fresh.
|
||||
- Top 50 requires at least 2 platforms (Tier >= 2), strictly verified and fresh.
|
||||
- Freshness Thresholds: YouTube <= 7d, Spotify <= 7d, Instagram <= 14d.
|
||||
- Integrity Gate: Artists with status_flag == 'FAILED_LOOKUP' are disqualified from YouTube credit.
|
||||
- Fail-closed: missing timestamp columns, stale records, or unverified statuses evaluate
|
||||
strictly to False and are logged to evidence_exclusions.
|
||||
- Preserves raw hiffi_xp_score and raw hiffi_xp_rank completely unchanged.
|
||||
- Produces consecutive, unique ranks in 'hiffi_xp_published_rank' (1 to N).
|
||||
"""
|
||||
) -> Union[pd.DataFrame, Tuple[pd.DataFrame, Dict[str, Any]]]:
|
||||
out = df.copy()
|
||||
ref_time = reference_time or pd.Timestamp.now(tz="UTC")
|
||||
|
||||
exclusions = [[] for _ in range(len(out))]
|
||||
|
||||
# 1. YouTube Validation (Score present + status not FAILED_LOOKUP + Freshness <= 7 days)
|
||||
status_col = out["status_flag"] if "status_flag" in out.columns else pd.Series("COMPLETE", index=out.index)
|
||||
has_yt_val = out["production_hiffi_score"].notna() if "production_hiffi_score" in out.columns else pd.Series(False, index=out.index)
|
||||
is_lookup_failed = status_col == "FAILED_LOOKUP"
|
||||
|
||||
if "youtube_updated_at" in out.columns:
|
||||
yt_fresh = out["youtube_updated_at"].apply(
|
||||
lambda x: self._is_signal_fresh(x, max_days=7.0, reference_time=ref_time)
|
||||
)
|
||||
if reference_time is None:
|
||||
ref_dt = datetime.now(timezone.utc)
|
||||
elif isinstance(reference_time, datetime):
|
||||
ref_dt = reference_time if reference_time.tzinfo else reference_time.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
yt_fresh = pd.Series(False, index=out.index)
|
||||
ref_dt = pd.to_datetime(reference_time, utc=True).to_pydatetime()
|
||||
|
||||
for idx, (val, failed, fresh) in enumerate(zip(has_yt_val, is_lookup_failed, yt_fresh)):
|
||||
if failed:
|
||||
exclusions[idx].append("FAILED_YOUTUBE_LOOKUP")
|
||||
elif val and not fresh:
|
||||
exclusions[idx].append("STALE_OR_MISSING_YOUTUBE_TIMESTAMP")
|
||||
max_staleness_days = 7.0
|
||||
|
||||
valid_yt = has_yt_val & (~is_lookup_failed) & yt_fresh
|
||||
def is_fresh(ts_val: Any) -> bool:
|
||||
if pd.isna(ts_val) or ts_val is None:
|
||||
return False
|
||||
try:
|
||||
dt = pd.to_datetime(ts_val, utc=True).to_pydatetime()
|
||||
diff = (ref_dt - dt).total_seconds() / 86400.0
|
||||
return 0.0 <= diff <= max_staleness_days
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# 2. Spotify Validation (Telemetry present + Freshness <= 7 days)
|
||||
has_sp_val = out["spotify_monthly_listeners"].notna() if "spotify_monthly_listeners" in out.columns else pd.Series(False, index=out.index)
|
||||
if "spotify_updated_at" in out.columns:
|
||||
sp_fresh = out["spotify_updated_at"].apply(
|
||||
lambda x: self._is_signal_fresh(x, max_days=7.0, reference_time=ref_time)
|
||||
)
|
||||
has_yt_col = "youtube_updated_at" in out.columns
|
||||
has_sp_col = "spotify_updated_at" in out.columns
|
||||
has_ig_col = "instagram_updated_at" in out.columns
|
||||
has_ver_col = "instagram_verification_status" in out.columns
|
||||
|
||||
yt_credits = []
|
||||
sp_credits = []
|
||||
ig_credits = []
|
||||
exclusions_list = []
|
||||
|
||||
for _, row in out.iterrows():
|
||||
row_exclusions = []
|
||||
|
||||
# 1. YouTube Evaluation
|
||||
status_flag = str(row.get("status_flag", "COMPLETE")).upper()
|
||||
if status_flag == "FAILED_LOOKUP":
|
||||
yt_ok = False
|
||||
row_exclusions.append("FAILED_YOUTUBE_LOOKUP")
|
||||
elif not has_yt_col:
|
||||
yt_ok = False
|
||||
row_exclusions.append("STALE_OR_MISSING_YOUTUBE_TIMESTAMP")
|
||||
else:
|
||||
sp_fresh = pd.Series(False, index=out.index)
|
||||
yt_ok = is_fresh(row.get("youtube_updated_at"))
|
||||
if not yt_ok:
|
||||
row_exclusions.append("STALE_OR_MISSING_YOUTUBE_TIMESTAMP")
|
||||
yt_credits.append(int(yt_ok))
|
||||
|
||||
for idx, (val, fresh) in enumerate(zip(has_sp_val, sp_fresh)):
|
||||
if val and not fresh:
|
||||
exclusions[idx].append("STALE_OR_MISSING_SPOTIFY_TIMESTAMP")
|
||||
valid_sp = has_sp_val & sp_fresh
|
||||
|
||||
# 3. Instagram Validation (Followers present + Approved Status + Freshness <= 14 days)
|
||||
has_ig_val = out["instagram_followers"].notna() if "instagram_followers" in out.columns else pd.Series(False, index=out.index)
|
||||
verified_statuses = {"AUTOMATED_VERIFIED", "MANUAL_VERIFIED"}
|
||||
|
||||
if "instagram_verification_status" in out.columns:
|
||||
ig_verified = out["instagram_verification_status"].isin(verified_statuses)
|
||||
# 2. Spotify Evaluation
|
||||
sp_val = row.get("spotify_monthly_listeners")
|
||||
has_sp_val = pd.notna(sp_val) and float(sp_val) > 0
|
||||
if not has_sp_col:
|
||||
sp_ok = False
|
||||
row_exclusions.append("STALE_OR_MISSING_SPOTIFY_TIMESTAMP")
|
||||
else:
|
||||
ig_verified = pd.Series(False, index=out.index)
|
||||
sp_ok = has_sp_val and is_fresh(row.get("spotify_updated_at"))
|
||||
if not sp_ok:
|
||||
row_exclusions.append("STALE_OR_MISSING_SPOTIFY_TIMESTAMP")
|
||||
sp_credits.append(int(sp_ok))
|
||||
|
||||
for idx, (val, ver) in enumerate(zip(has_ig_val, ig_verified)):
|
||||
if val and not ver:
|
||||
exclusions[idx].append("UNVERIFIED_INSTAGRAM_IDENTITY")
|
||||
# 3. Instagram Evaluation
|
||||
ig_val = row.get("instagram_followers")
|
||||
has_ig_val = pd.notna(ig_val) and float(ig_val) > 0
|
||||
|
||||
if "instagram_updated_at" in out.columns:
|
||||
ig_fresh = out["instagram_updated_at"].apply(
|
||||
lambda x: self._is_signal_fresh(x, max_days=14.0, reference_time=ref_time)
|
||||
)
|
||||
if not has_ver_col:
|
||||
ig_ver_ok = False
|
||||
else:
|
||||
ig_fresh = pd.Series(False, index=out.index)
|
||||
ver_status = str(row.get("instagram_verification_status", "")).upper()
|
||||
ig_ver_ok = ver_status in ("AUTOMATED_VERIFIED", "MANUAL_VERIFIED", "VERIFIED")
|
||||
|
||||
for idx, (val, ver, fresh) in enumerate(zip(has_ig_val, ig_verified, ig_fresh)):
|
||||
if val and ver and not fresh:
|
||||
exclusions[idx].append("STALE_OR_MISSING_INSTAGRAM_TIMESTAMP")
|
||||
if not has_ig_col:
|
||||
ig_fresh = False
|
||||
row_exclusions.append("STALE_OR_MISSING_INSTAGRAM_TIMESTAMP")
|
||||
else:
|
||||
ig_fresh = is_fresh(row.get("instagram_updated_at"))
|
||||
if not ig_fresh:
|
||||
row_exclusions.append("STALE_OR_MISSING_INSTAGRAM_TIMESTAMP")
|
||||
|
||||
valid_ig = has_ig_val & ig_verified & ig_fresh
|
||||
ig_ok = has_ig_val and ig_ver_ok and ig_fresh
|
||||
ig_credits.append(int(ig_ok))
|
||||
|
||||
out["platform_evidence_tier"] = (
|
||||
valid_yt.astype(int) + valid_sp.astype(int) + valid_ig.astype(int)
|
||||
)
|
||||
out["evidence_exclusions"] = [";".join(e) if e else "NONE" for e in exclusions]
|
||||
exclusions_list.append(";".join(row_exclusions) if row_exclusions else "NONE")
|
||||
|
||||
# 4. Deterministic sorting by raw composite score
|
||||
sort_cols = ["hiffi_xp_score", "production_hiffi_score", "spotify_monthly_listeners"]
|
||||
present_sort_cols = [c for c in sort_cols if c in out.columns]
|
||||
pool = out.sort_values(
|
||||
by=present_sort_cols,
|
||||
ascending=[False] * len(present_sort_cols)
|
||||
).copy()
|
||||
out["gated_yt_credit"] = yt_credits
|
||||
out["gated_sp_credit"] = sp_credits
|
||||
out["gated_ig_credit"] = ig_credits
|
||||
out["platform_evidence_tier"] = [y + s + i for y, s, i in zip(yt_credits, sp_credits, ig_credits)]
|
||||
out["evidence_exclusions"] = exclusions_list
|
||||
|
||||
assigned_indices = []
|
||||
shortfall_audit = {"top_10_shortfall": False, "top_50_shortfall": False}
|
||||
|
||||
# Pass 1: Select Top 10 from Tier 3 candidates
|
||||
tier3_candidates = pool[pool["platform_evidence_tier"] == 3]
|
||||
top_10_indices = tier3_candidates.head(10).index.tolist()
|
||||
assigned_indices.extend(top_10_indices)
|
||||
|
||||
if len(assigned_indices) < min(10, len(pool)):
|
||||
shortfall_audit["top_10_shortfall"] = True
|
||||
remaining_pool = pool.drop(index=assigned_indices)
|
||||
shortfall = min(10 - len(assigned_indices), len(remaining_pool))
|
||||
backfill = remaining_pool[remaining_pool["platform_evidence_tier"] >= 2].head(shortfall)
|
||||
assigned_indices.extend(backfill.index.tolist())
|
||||
|
||||
if len(assigned_indices) < min(10, len(pool)):
|
||||
remaining_pool = pool.drop(index=assigned_indices)
|
||||
shortfall = min(10 - len(assigned_indices), len(remaining_pool))
|
||||
assigned_indices.extend(remaining_pool.head(shortfall).index.tolist())
|
||||
|
||||
# Pass 2: Select Positions 11 to 50 from Tier >= 2 candidates
|
||||
remaining_pool = pool.drop(index=assigned_indices)
|
||||
needed_for_50 = min(40, len(pool) - len(assigned_indices))
|
||||
tier2_plus_candidates = remaining_pool[remaining_pool["platform_evidence_tier"] >= 2]
|
||||
top_50_fill = tier2_plus_candidates.head(needed_for_50).index.tolist()
|
||||
assigned_indices.extend(top_50_fill)
|
||||
|
||||
if len(assigned_indices) < min(50, len(pool)):
|
||||
shortfall_audit["top_50_shortfall"] = True
|
||||
remaining_pool = pool.drop(index=assigned_indices)
|
||||
shortfall = min(50 - len(assigned_indices), len(remaining_pool))
|
||||
assigned_indices.extend(remaining_pool.head(shortfall).index.tolist())
|
||||
|
||||
# Pass 3: Positions 51+ with all remaining artists strictly by score order
|
||||
final_remaining = pool.drop(index=assigned_indices)
|
||||
assigned_indices.extend(final_remaining.index.tolist())
|
||||
|
||||
# 5. Assign consecutive published ranks
|
||||
out["hiffi_xp_published_rank"] = pd.NA
|
||||
rank_series = pd.Series(
|
||||
range(1, len(assigned_indices) + 1),
|
||||
index=assigned_indices,
|
||||
dtype="Int64"
|
||||
)
|
||||
out.loc[assigned_indices, "hiffi_xp_published_rank"] = rank_series
|
||||
out["hiffi_xp_published_rank"] = out["hiffi_xp_published_rank"].astype(int)
|
||||
if "hiffi_xp_published_rank" not in out.columns:
|
||||
if "hiffi_xp_score" in out.columns:
|
||||
out["hiffi_xp_published_rank"] = out["hiffi_xp_score"].rank(ascending=False, method="min").astype(int)
|
||||
else:
|
||||
out["hiffi_xp_published_rank"] = list(range(1, len(out) + 1))
|
||||
|
||||
if return_metadata:
|
||||
return out, shortfall_audit
|
||||
top_10 = out.head(10)
|
||||
top_50 = out.head(50)
|
||||
metadata = {
|
||||
"total_records": len(out),
|
||||
"top_10_shortfall": bool((top_10["platform_evidence_tier"] < 3).any()),
|
||||
"top_50_shortfall": bool((top_50["platform_evidence_tier"] < 3).any()),
|
||||
"tier_distribution": out["platform_evidence_tier"].value_counts().to_dict(),
|
||||
"reference_time": ref_dt.isoformat(),
|
||||
}
|
||||
return out, metadata
|
||||
|
||||
return out
|
||||
|
||||
def detect_metric_anomalies(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
out = df.copy()
|
||||
flags = []
|
||||
|
||||
for _, row in out.iterrows():
|
||||
row_flags = []
|
||||
views = float(row.get("lifetime_views") or 0.0)
|
||||
ig = float(row.get("instagram_followers") or 0.0)
|
||||
sp_month = float(row.get("spotify_monthly_listeners") or 0.0)
|
||||
sp_change = float(row.get("spotify_daily_change") or 0.0)
|
||||
|
||||
if views > 0 and ig > 0:
|
||||
if (ig / views) > 10.0:
|
||||
row_flags.append("SUSPICIOUS_IG_TO_YT_RATIO")
|
||||
|
||||
if sp_month > 0 and sp_change > 0:
|
||||
if (sp_change / sp_month) > 0.20:
|
||||
row_flags.append("EXTREME_SPOTIFY_VELOCITY_RATIO")
|
||||
|
||||
flags.append(";".join(row_flags) if row_flags else "CLEAN")
|
||||
|
||||
out["anomaly_flags"] = flags
|
||||
return out
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 97 KiB After Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 35 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 29 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 33 KiB After Width: | Height: | Size: 33 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 9.4 KiB After Width: | Height: | Size: 26 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 37 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 150 KiB After Width: | Height: | Size: 0 B |
Binary file not shown.
|
Before Width: | Height: | Size: 29 KiB After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 39 KiB After Width: | Height: | Size: 39 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 48 KiB After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 30 KiB After Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 47 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 10 KiB After Width: | Height: | Size: 26 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 30 KiB After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 7.0 KiB After Width: | Height: | Size: 22 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 271 KiB After Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 33 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 23 KiB After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 41 KiB After Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 9.2 KiB After Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 595 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 27 KiB After Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 10 KiB After Width: | Height: | Size: 29 KiB |
Binary file not shown.
+501
-501
File diff suppressed because it is too large.
Load diff
+880
-880
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,106 @@
|
||||
import sqlite3
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
def run_telemetry_drift_audit(
|
||||
db_path: str = "data/hiffi_catalog.db",
|
||||
max_daily_listener_drop_pct: float = 25.0,
|
||||
max_growth_spike_pct: float = 500.0
|
||||
) -> dict:
|
||||
"""
|
||||
Evaluates cohort telemetry against operational drift thresholds:
|
||||
1. Catastrophic Drops (>25% daily decline on high-tier artists).
|
||||
2. Anomalous Surges (>500% daily velocity requiring corroboration).
|
||||
3. Governance Violations (Hak non-null regressions).
|
||||
4. Null Velocity Drift on populated accounts.
|
||||
"""
|
||||
con = sqlite3.connect(db_path)
|
||||
query = """
|
||||
SELECT
|
||||
r.catalog_rank,
|
||||
a.artist_id,
|
||||
a.artist_name,
|
||||
a.youtube_url,
|
||||
a.spotify_url,
|
||||
a.instagram_url,
|
||||
t.subscribers_count,
|
||||
t.spotify_monthly_listeners,
|
||||
t.instagram_followers,
|
||||
t.daily_view_velocity,
|
||||
t.growth_velocity_7d,
|
||||
t.momentum_score,
|
||||
t.spotify_daily_change
|
||||
FROM rankings r
|
||||
JOIN artists a ON r.artist_id = a.artist_id
|
||||
JOIN telemetry_metrics t ON a.artist_id = t.artist_id
|
||||
WHERE r.catalog_rank BETWEEN 1 AND 500
|
||||
ORDER BY r.catalog_rank ASC
|
||||
"""
|
||||
df = pd.read_sql_query(query, con)
|
||||
con.close()
|
||||
|
||||
anomalies = []
|
||||
|
||||
print("=" * 80)
|
||||
print("HIFFI AUTOMATED DRIFT & VELOCITY AUDIT")
|
||||
print("=" * 80)
|
||||
|
||||
# 1. Governance Drift Check (Hak Tier-1)
|
||||
hak = df[df["artist_id"] == "@haknyc"]
|
||||
if not hak.empty:
|
||||
hak_row = hak.iloc[0]
|
||||
if pd.notna(hak_row["spotify_url"]) or pd.notna(hak_row["spotify_monthly_listeners"]) or \
|
||||
pd.notna(hak_row["instagram_url"]) or pd.notna(hak_row["instagram_followers"]):
|
||||
anomalies.append({
|
||||
"artist": "Hak",
|
||||
"type": "GOVERNANCE_BREACH",
|
||||
"detail": "Hak has non-null values imputed for Spotify/Instagram"
|
||||
})
|
||||
|
||||
# 2. Daily Spotify Velocity Drift
|
||||
# Calculate % delta: (daily_change / (monthly_listeners - daily_change)) * 100
|
||||
active_sp = df[df["spotify_monthly_listeners"].notna() & (df["spotify_monthly_listeners"] > 0)].copy()
|
||||
prev_listeners = active_sp["spotify_monthly_listeners"] - active_sp["spotify_daily_change"]
|
||||
active_sp["pct_daily_change"] = (active_sp["spotify_daily_change"] / prev_listeners.replace(0, np.nan)) * 100
|
||||
|
||||
severe_drops = active_sp[active_sp["pct_daily_change"] < -max_daily_listener_drop_pct]
|
||||
for _, row in severe_drops.iterrows():
|
||||
anomalies.append({
|
||||
"artist": row["artist_name"],
|
||||
"type": "CATASTROPHIC_DROP",
|
||||
"detail": f"Daily listener drop of {row['pct_daily_change']:.2f}% (Loss: {int(row['spotify_daily_change'])})"
|
||||
})
|
||||
|
||||
anomalous_spikes = active_sp[active_sp["pct_daily_change"] > max_growth_spike_pct]
|
||||
for _, row in anomalous_spikes.iterrows():
|
||||
anomalies.append({
|
||||
"artist": row["artist_name"],
|
||||
"type": "UNVERIFIED_SPIKE",
|
||||
"detail": f"Daily listener surge of {row['pct_daily_change']:.2f}% (Gain: {int(row['spotify_daily_change'])})"
|
||||
})
|
||||
|
||||
# 3. Canonical Schema Drift (Broken URL formats)
|
||||
yt_bad = df[~df["youtube_url"].fillna("").str.contains(r"^https://www\.youtube\.com/channel/UC[a-zA-Z0-9_-]{22}$", regex=True)]
|
||||
for _, row in yt_bad.iterrows():
|
||||
anomalies.append({
|
||||
"artist": row["artist_name"],
|
||||
"type": "SCHEMA_DRIFT_YOUTUBE",
|
||||
"detail": f"Non-canonical YouTube URL: {row['youtube_url']}"
|
||||
})
|
||||
|
||||
print(f"Total Cohort Size Audited: {len(df)}")
|
||||
print(f"Total Drift Anomalies Flagged: {len(anomalies)}")
|
||||
|
||||
if anomalies:
|
||||
print("\n[CIRCUIT BREAKER TRIGGERED] Anomalies detected:")
|
||||
for idx, a in enumerate(anomalies, 1):
|
||||
print(f" {idx}. [{a['type']}] {a['artist']}: {a['detail']}")
|
||||
print("\nAction: Synchronization to production aborted.")
|
||||
return {"circuit_breaker": "TRIPPED", "anomalies": anomalies}
|
||||
else:
|
||||
print("[CIRCUIT BREAKER: CLEAR] No drift detected. Telemetry velocities healthy.")
|
||||
print("Action: Safe to synchronize with live production API.")
|
||||
return {"circuit_breaker": "PASS", "anomalies": []}
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_telemetry_drift_audit()
|
||||
@@ -117,3 +117,4 @@ def main():
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in new issue
Block a user