checkpoint: working 10-artist batch with log-normalized ranking

core/ranking_engine.py

@@ -0,0 +1,83 @@
+import numpy as np
+import pandas as pd
+from typing import Dict, Any
+
+class RankingEngine:
+    """
+    Computes equitable artist rankings using Log Transformation (log1p)
+    and Min-Max Scaling across skewed social/streaming distribution metrics.
+    """
+    DEFAULT_WEIGHTS = {
+        "norm_subscribers": 0.30,
+        "norm_lifetime_views": 0.25,
+        "norm_recent_avg_views": 0.25,
+        "norm_video_count": 0.10,
+        "norm_momentum_30d": 0.10
+    }
+
+    def __init__(self, weights: Dict[str, float] = None):
+        self.weights = weights or self.DEFAULT_WEIGHTS
+
+    def normalize_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Applies log10(1 + x) and Min-Max scaling to compress power-law tails."""
+        df_scored = df.copy()
+
+        metrics_to_transform = [
+            ("yt_subscribers", "norm_subscribers"),
+            ("yt_lifetime_views", "norm_lifetime_views"),
+            ("yt_video_count", "norm_video_count"),
+            ("yt_recent_avg_views", "norm_recent_avg_views"),
+            ("yt_momentum_30d", "norm_momentum_30d")
+        ]
+
+        for raw_col, norm_col in metrics_to_transform:
+            if raw_col not in df_scored.columns:
+                df_scored[norm_col] = 0.0
+                continue
+
+            # Step 1: Handle NaNs and negative values, then apply Log10(1 + x)
+            series = pd.to_numeric(df_scored[raw_col], errors="coerce").fillna(0).clip(lower=0)
+            log_series = np.log10(1 + series)
+
+            min_val = log_series.min()
+            max_val = log_series.max()
+
+            # Step 2: Min-Max Normalization to scale [0.0, 1.0]
+            if max_val > min_val:
+                df_scored[norm_col] = (log_series - min_val) / (max_val - min_val)
+            else:
+                df_scored[norm_col] = 0.0
+
+        # Step 3: Compute weighted base score
+        df_scored["base_rank_score"] = sum(
+            df_scored[col] * weight for col, weight in self.weights.items()
+        )
+
+        # Step 4: Multiply by verification confidence to penalize unverified or fake profiles
+        # Artists with verified MusicBrainz/Wikidata and legitimate channels get full weight
+        confidence = df_scored.get("composite_confidence", pd.Series(1.0, index=df_scored.index))
+        # Ensure a minimum floor of 0.2 so emerging artists are ranked on data, not dropped completely
+        confidence_multiplier = 0.2 + (0.8 * confidence)
+        
+        # Step 5: Final HIFFI Score on a clean 0 to 100 scale
+        df_scored["hiffi_score"] = (df_scored["base_rank_score"] * confidence_multiplier * 100).round(2)
+        
+        # Sort and assign explicit rank ordinal
+        df_scored = df_scored.sort_values(by="hiffi_score", ascending=False).reset_index(drop=True)
+        df_scored["hiffi_rank"] = df_scored.index + 1
+
+        return df_scored
+
+if __name__ == "__main__":
+    # Test script with dummy mock distribution
+    sample_data = pd.DataFrame([
+        {"Name": "Global Superstar", "yt_subscribers": 15000000, "yt_lifetime_views": 8500000000, "yt_video_count": 120, "yt_recent_avg_views": 2500000, "yt_momentum_30d": 1.2, "composite_confidence": 0.95},
+        {"Name": "Mid-tier Rising", "yt_subscribers": 250000, "yt_lifetime_views": 45000000, "yt_video_count": 45, "yt_recent_avg_views": 180000, "yt_momentum_30d": 5.4, "composite_confidence": 0.85},
+        {"Name": "Emerging Talent", "yt_subscribers": 12000, "yt_lifetime_views": 850000, "yt_video_count": 14, "yt_recent_avg_views": 45000, "yt_momentum_30d": 12.8, "composite_confidence": 0.60},
+        {"Name": "Dormant / Inactive", "yt_subscribers": 500, "yt_lifetime_views": 15000, "yt_video_count": 3, "yt_recent_avg_views": 50, "yt_momentum_30d": 0.0, "composite_confidence": 0.35}
+    ])
+    
+    engine = RankingEngine()
+    ranked = engine.normalize_metrics(sample_data)
+    print("Ranked Test Artists with Log & Min-Max Transformation:")
+    print(ranked[["hiffi_rank", "Name", "hiffi_score", "norm_subscribers", "norm_lifetime_views"]])