srilekha/hiffi-artist-intelligence
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: scale HIFFI pipeline to 500 artists with dual BVI ranking, dark dashboard, and Excel audit export

Browse Source
This commit is contained in:
Srilekha committed 2026-10-22 17:20:44 +05:30
1 parent 5d441ffe7c
commit 9710cf9948
14 files changed
+3017 -358
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

core/ranking_engine.py
+155 -71
View File
@@ -1,83 +1,167 @@
"""
core/ranking_engine.py
Ranks artists according to HIFFI formula stages:
- Recent Momentum (35%)
- View Velocity (30%)
- Audience Reach (20%)
- Consistency (15%)
Calculates Breakout Velocity Index (BVI, 0-100) and breakout_rank
with a 10,000 subscriber floor to prevent phantom accounts from ranking.
"""
import numpy as np
import pandas as pd
from typing import Dict, Any
class RankingEngine:
"""
Computes equitable artist rankings using Log Transformation (log1p)
and Min-Max Scaling across skewed social/streaming distribution metrics.
"""
DEFAULT_WEIGHTS = {
"norm_subscribers": 0.30,
"norm_lifetime_views": 0.25,
"norm_recent_avg_views": 0.25,
"norm_video_count": 0.10,
"norm_momentum_30d": 0.10
}
def __init__(self, weights: Dict[str, float] = None):
self.weights = weights or self.DEFAULT_WEIGHTS
class HIFFIRankingEngine:
def normalize_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
"""Applies log10(1 + x) and Min-Max scaling to compress power-law tails."""
df_scored = df.copy()
@staticmethod
def _log_scale(series: pd.Series) -> pd.Series:
valid = series.dropna()
log_vals = np.log10(1.0 + np.maximum(valid, 0.0))
s_min, s_max = log_vals.min(), log_vals.max()
if np.isclose(s_max, s_min) or pd.isna(s_min):
return pd.Series(0.0, index=series.index)
scaled = (log_vals - s_min) / (s_max - s_min + 1e-9)
return scaled.reindex(series.index)
metrics_to_transform = [
("yt_subscribers", "norm_subscribers"),
("yt_lifetime_views", "norm_lifetime_views"),
("yt_video_count", "norm_video_count"),
("yt_recent_avg_views", "norm_recent_avg_views"),
("yt_momentum_30d", "norm_momentum_30d")
]
@staticmethod
def _min_max_scale(series: pd.Series) -> pd.Series:
valid = series.dropna()
s_min, s_max = valid.min(), valid.max()
if np.isclose(s_max, s_min) or pd.isna(s_min):
return pd.Series(0.0, index=series.index)
scaled = (valid - s_min) / (s_max - s_min + 1e-9)
return scaled.reindex(series.index)
for raw_col, norm_col in metrics_to_transform:
if raw_col not in df_scored.columns:
df_scored[norm_col] = 0.0
continue
def calculate_breakout_velocity(self, df: pd.DataFrame) -> pd.Series:
"""
Computes the Breakout Velocity Index (BVI, 0 - 100).
Requires at least 10,000 subscribers and positive velocity to qualify.
"""
out_bvi = pd.Series(np.nan, index=df.index, dtype=float)
# Step 1: Handle NaNs and negative values, then apply Log10(1 + x)
series = pd.to_numeric(df_scored[raw_col], errors="coerce").fillna(0).clip(lower=0)
log_series = np.log10(1 + series)
min_val = log_series.min()
max_val = log_series.max()
# Step 2: Min-Max Normalization to scale [0.0, 1.0]
if max_val > min_val:
df_scored[norm_col] = (log_series - min_val) / (max_val - min_val)
else:
df_scored[norm_col] = 0.0
# Step 3: Compute weighted base score
df_scored["base_rank_score"] = sum(
df_scored[col] * weight for col, weight in self.weights.items()
# Qualification Gate: Must have real baseline audience (>= 10,000 subs)
valid_mask = (
df["subscriber_count"].notna()
& (df["subscriber_count"] >= 10000)
& df["daily_view_velocity"].notna()
& (df["daily_view_velocity"] > 0.0)
& (df["status_flag"] != "INACTIVE_OR_DORMANT")
& (df["status_flag"] != "FAILED_LOOKUP")
)
# Step 4: Multiply by verification confidence to penalize unverified or fake profiles
# Artists with verified MusicBrainz/Wikidata and legitimate channels get full weight
confidence = df_scored.get("composite_confidence", pd.Series(1.0, index=df_scored.index))
# Ensure a minimum floor of 0.2 so emerging artists are ranked on data, not dropped completely
confidence_multiplier = 0.2 + (0.8 * confidence)
# Step 5: Final HIFFI Score on a clean 0 to 100 scale
df_scored["hiffi_score"] = (df_scored["base_rank_score"] * confidence_multiplier * 100).round(2)
# Sort and assign explicit rank ordinal
df_scored = df_scored.sort_values(by="hiffi_score", ascending=False).reset_index(drop=True)
df_scored["hiffi_rank"] = df_scored.index + 1
if not valid_mask.any():
return out_bvi
return df_scored
subset = df[valid_mask]
subs = subset["subscriber_count"].astype(float)
daily_vel = subset["daily_view_velocity"].astype(float)
momentum = subset["momentum_30d"].fillna(0.0).clip(lower=-100.0, upper=300.0)
if __name__ == "__main__":
# Test script with dummy mock distribution
sample_data = pd.DataFrame([
{"Name": "Global Superstar", "yt_subscribers": 15000000, "yt_lifetime_views": 8500000000, "yt_video_count": 120, "yt_recent_avg_views": 2500000, "yt_momentum_30d": 1.2, "composite_confidence": 0.95},
{"Name": "Mid-tier Rising", "yt_subscribers": 250000, "yt_lifetime_views": 45000000, "yt_video_count": 45, "yt_recent_avg_views": 180000, "yt_momentum_30d": 5.4, "composite_confidence": 0.85},
{"Name": "Emerging Talent", "yt_subscribers": 12000, "yt_lifetime_views": 850000, "yt_video_count": 14, "yt_recent_avg_views": 45000, "yt_momentum_30d": 12.8, "composite_confidence": 0.60},
{"Name": "Dormant / Inactive", "yt_subscribers": 500, "yt_lifetime_views": 15000, "yt_video_count": 3, "yt_recent_avg_views": 50, "yt_momentum_30d": 0.0, "composite_confidence": 0.35}
])
engine = RankingEngine()
ranked = engine.normalize_metrics(sample_data)
print("Ranked Test Artists with Log & Min-Max Transformation:")
print(ranked[["hiffi_rank", "Name", "hiffi_score", "norm_subscribers", "norm_lifetime_views"]])
# 1. Audience Velocity Ratio: Views per day relative to subscriber base
v_aud = daily_vel / np.maximum(subs, 1000.0)
# 2. Scaled Momentum Factor: Map [-100, 300] to [0, 1]
m_signal = (momentum + 100.0) / 400.0
# 3. Log-compress velocity to manage power-law peaks
log_v_aud = np.log10(1.0 + v_aud)
# 4. Emergence Dampener: inverse log weight to favor emerging accounts
w_emerge = 1.0 / (1.0 + np.log10(1.0 + np.maximum(subs, 1.0)))
raw_bvi = (0.65 * log_v_aud + 0.35 * m_signal) * w_emerge
# Min-max scale to 0-100 across valid records
bvi_min = raw_bvi.min()
bvi_max = raw_bvi.max()
if np.isclose(bvi_max, bvi_min):
scaled_bvi = pd.Series(50.0, index=subset.index)
else:
scaled_bvi = ((raw_bvi - bvi_min) / (bvi_max - bvi_min + 1e-9)) * 100.0
out_bvi.loc[valid_mask] = scaled_bvi.round(2)
return out_bvi
def score_and_rank(
self,
df: pd.DataFrame,
min_confidence: float = 0.50,
) -> pd.DataFrame:
out = df.copy()
# Handle confidence score defensively
if "confidence_score" in out.columns:
out["confidence_score"] = pd.to_numeric(
out["confidence_score"].apply(
lambda x: x.get("composite_confidence", 0.0) if isinstance(x, dict) else x
),
errors="coerce"
).fillna(0.0)
# Compute BVI with the 10,000 sub floor
out["breakout_velocity_index"] = self.calculate_breakout_velocity(out)
# Verification & completeness gate
has_baseline = (
out["subscriber_count"].notna()
& out["lifetime_views"].notna()
& (out["status_flag"] != "FAILED_LOOKUP")
)
out["is_rank_eligible"] = (
has_baseline
& (out["confidence_score"] >= min_confidence)
& (out["status_flag"].isin(["COMPLETE", "CATALOG_ACTIVE_NO_RECENT_UPLOAD"]))
)
# 1. Recent Momentum (35%)
filled_momentum = out["momentum_30d"].fillna(0.0).clip(lower=-100.0, upper=300.0)
score_momentum = self._min_max_scale(filled_momentum)
# 2. View Velocity (30%)
filled_velocity = out["daily_view_velocity"].fillna(0.0)
score_velocity = self._log_scale(filled_velocity)
# 3. Audience Reach (20%)
log_subs = self._log_scale(out["subscriber_count"].fillna(0.0))
log_lifetime = self._log_scale(out["lifetime_views"].fillna(0.0))
score_reach = 0.60 * log_subs + 0.40 * log_lifetime
# 4. Consistency (15%)
score_consistency = self._log_scale(out["video_count"].fillna(0.0))
# Composite HIFFI Score (0 - 100)
hiffi_raw = (
0.35 * score_momentum
+ 0.30 * score_velocity
+ 0.20 * score_reach
+ 0.15 * score_consistency
) * 100.0
conf_multiplier = 0.8 + 0.2 * out["confidence_score"].clip(0.0, 1.0)
out["hiffi_score"] = (hiffi_raw * conf_multiplier).round(2)
# Primary HIFFI Rank
eligible_mask = out["is_rank_eligible"]
out["hiffi_rank"] = pd.NA
out.loc[eligible_mask, "hiffi_rank"] = (
out.loc[eligible_mask, "hiffi_score"]
.rank(ascending=False, method="min")
.astype(int)
)
# Breakout Rank (only for eligible artists that have a valid BVI)
bvi_eligible_mask = eligible_mask & out["breakout_velocity_index"].notna()
out["breakout_rank"] = pd.NA
out.loc[bvi_eligible_mask, "breakout_rank"] = (
out.loc[bvi_eligible_mask, "breakout_velocity_index"]
.rank(ascending=False, method="min")
.astype(int)
)
return out.sort_values(
by=["is_rank_eligible", "hiffi_score"], ascending=[False, False]
).reset_index(drop=True)