feat(release): Phase 1-5 multi-platform pipeline, CI workflow, regression auditor, and orchestrator

core/ranking_engine.py

@@ -8,12 +8,15 @@ Ranks artists according to HIFFI formula stages:
   * Spotify Monthly Listeners (25%)
   * Spotify Daily Change / Velocity (5%)
   * Instagram Followers (20%)
-  With zero-imputation compliance and proportional weight redistribution.
+  With zero-imputation compliance, proportional weight redistribution,
+  and effective weight serialization.
 - Publication Eligibility Gates: Decoupled public chart assigning consecutive ranks
-  while enforcing multi-platform evidence tiers for Top 10 and Top 50.
+  while enforcing multi-platform evidence tiers (Top 10 = Tier 3, Top 50 = Tier >= 2)
+  conditioned on strict identity verification and UTC data freshness thresholds.
+  Fail-closed on missing timestamp columns or unverified statuses.
 """
 
-from typing import Optional
+from typing import Optional, Tuple, Union
 import numpy as np
 import pandas as pd
 
@@ -39,6 +42,27 @@ class HIFFIRankingEngine:
         scaled = (valid - s_min) / (s_max - s_min + 1e-9)
         return scaled.reindex(series.index)
 
+    @staticmethod
+    def _is_signal_fresh(
+        timestamp_val: object,
+        max_days: float,
+        reference_time: Optional[pd.Timestamp] = None
+    ) -> bool:
+        """
+        Parses UTC ISO 8601 string or pd.Timestamp and checks staleness.
+        Fails closed (returns False) on missing, null, or unparseable timestamps.
+        """
+        if pd.isna(timestamp_val):
+            return False
+
+        try:
+            ts = pd.to_datetime(timestamp_val, utc=True)
+            ref = reference_time or pd.Timestamp.now(tz="UTC")
+            age_days = (ref - ts).total_seconds() / 86400.0
+            return 0.0 <= age_days <= max_days
+        except Exception:
+            return False
+
     def calculate_breakout_velocity(self, df: pd.DataFrame) -> pd.Series:
         """
         Computes the Breakout Velocity Index (BVI, 0 - 100).
@@ -153,7 +177,7 @@ class HIFFIRankingEngine:
 
     def calculate_hiffi_xp(self, df: pd.DataFrame) -> pd.DataFrame:
         """
-        Calculates HIFFI-XP composite multi-platform score and rank.
+        Calculates HIFFI-XP composite multi-platform score, raw rank, and effective weights.
         Enforces proportional weight redistribution when signals are missing (NaN).
         Weights: YouTube (0.50), Spotify Monthly (0.25), Spotify Velocity (0.05), Instagram (0.20).
         """
@@ -207,14 +231,21 @@ class HIFFIRankingEngine:
         weight_sum = weight_sum.replace(0.0, np.nan)
         normalized_weights = allocated_weights.divide(weight_sum, axis=0)
 
+        # Explainability: Format explicit percentage breakdown per artist
+        def format_weights(row: pd.Series) -> str:
+            parts = [f"{col}:{val*100.0:.1f}%" for col, val in row.items() if val > 0]
+            return ",".join(parts) if parts else "NONE"
+
+        out["effective_weights"] = normalized_weights.apply(format_weights, axis=1)
+
         component_scores = (signals.fillna(0.0) * normalized_weights).sum(axis=1) * 100.0
         out["hiffi_xp_score"] = component_scores.round(2)
         out.loc[weight_sum.isna(), "hiffi_xp_score"] = np.nan
 
-        # Deterministic tie-breaking for raw mathematical ranking
+        # Deterministic tie-breaking for raw continuous ranking
         sort_cols = ["hiffi_xp_score", "production_hiffi_score", "spotify_monthly_listeners"]
         present_sort = [c for c in sort_cols if c in out.columns]
-        
+
         valid_mask = out["hiffi_xp_score"].notna()
         sorted_indices = (
             out[valid_mask]
@@ -233,40 +264,123 @@ class HIFFIRankingEngine:
 
         return out
 
+    def detect_metric_anomalies(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Flags suspicious metric patterns without altering raw scores:
+        - SUSPICIOUS_IG_TO_YT_RATIO: IG followers > 10x YouTube lifetime views.
+        - EXTREME_SPOTIFY_VELOCITY_RATIO: Spotify daily change > 25% of monthly listeners.
+        - Populates 'anomaly_flags' column (semicolon-delimited or 'CLEAN').
+        """
+        out = df.copy()
+        flags = [[] for _ in range(len(out))]
+
+        # 1. Instagram to YouTube Lifetime Ratio Check
+        if "instagram_followers" in out.columns and "lifetime_views" in out.columns:
+            ig = pd.to_numeric(out["instagram_followers"], errors="coerce")
+            yt_views = pd.to_numeric(out["lifetime_views"], errors="coerce")
+            suspicious_ratio = (ig.notna() & yt_views.notna() & (yt_views > 0) & (ig > 10.0 * yt_views))
+            for idx, is_suspicious in enumerate(suspicious_ratio):
+                if is_suspicious:
+                    flags[idx].append("SUSPICIOUS_IG_TO_YT_RATIO")
+
+        # 2. Spotify Velocity to Monthly Listeners Check
+        if "spotify_daily_change" in out.columns and "spotify_monthly_listeners" in out.columns:
+            sp_change = pd.to_numeric(out["spotify_daily_change"], errors="coerce")
+            sp_monthly = pd.to_numeric(out["spotify_monthly_listeners"], errors="coerce")
+            extreme_velocity = (
+                sp_change.notna()
+                & sp_monthly.notna()
+                & (sp_monthly > 1000)
+                & (sp_change.abs() > 0.25 * sp_monthly)
+            )
+            for idx, is_extreme in enumerate(extreme_velocity):
+                if is_extreme:
+                    flags[idx].append("EXTREME_SPOTIFY_VELOCITY_RATIO")
+
+        out["anomaly_flags"] = [";".join(f) if f else "CLEAN" for f in flags]
+        return out
+
     def apply_publication_eligibility_gates(
         self,
         df: pd.DataFrame,
+        reference_time: Optional[pd.Timestamp] = None,
         return_metadata: bool = False
-    ) -> pd.DataFrame:
+    ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, dict]]:
         """
         Constructs a publication ranking layer decoupled from raw scores:
-        - Top 10 requires all 3 platforms (Tier 3), strictly verified.
-        - Top 50 requires at least 2 platforms (Tier >= 2), strictly verified.
-        - Unverified or unindexed Instagram accounts do NOT contribute to evidence tiers.
-        - Fail-closed: missing verification column evaluates strictly to False.
-        - Explicitly tracks shortfalls and degrades gracefully without rank gaps or duplicate integers.
+        - Top 10 requires all 3 platforms (Tier 3), strictly verified and fresh.
+        - Top 50 requires at least 2 platforms (Tier >= 2), strictly verified and fresh.
+        - Freshness Thresholds: YouTube <= 7d, Spotify <= 7d, Instagram <= 14d.
+        - Fail-closed: missing timestamp columns or unverified statuses evaluate
+          strictly to False and are logged to evidence_exclusions.
+        - Preserves raw hiffi_xp_score and raw hiffi_xp_rank completely unchanged.
+        - Produces consecutive, unique ranks in 'hiffi_xp_published_rank' (1 to N).
         """
         out = df.copy()
+        ref_time = reference_time or pd.Timestamp.now(tz="UTC")
 
-        # 1. Strict Signal Verification
-        has_yt = out["production_hiffi_score"].notna()
-        has_sp = out["spotify_monthly_listeners"].notna()
+        exclusions = [[] for _ in range(len(out))]
 
-        # Fail-closed Instagram verification check
-        verified_ig_statuses = {"AUTOMATED_VERIFIED", "MANUAL_VERIFIED"}
-        if "instagram_verification_status" in out.columns:
-            has_ig = (
-                out["instagram_followers"].notna()
-                & out["instagram_verification_status"].isin(verified_ig_statuses)
+        # 1. YouTube Validation (Score present + Freshness <= 7 days)
+        has_yt_val = out["production_hiffi_score"].notna()
+        if "youtube_updated_at" in out.columns:
+            yt_fresh = out["youtube_updated_at"].apply(
+                lambda x: self._is_signal_fresh(x, max_days=7.0, reference_time=ref_time)
             )
         else:
-            has_ig = pd.Series(False, index=out.index)
+            yt_fresh = pd.Series(False, index=out.index)
+
+        for idx, (val, fresh) in enumerate(zip(has_yt_val, yt_fresh)):
+            if val and not fresh:
+                exclusions[idx].append("STALE_OR_MISSING_YOUTUBE_TIMESTAMP")
+        valid_yt = has_yt_val & yt_fresh
+
+        # 2. Spotify Validation (Telemetry present + Freshness <= 7 days)
+        has_sp_val = out["spotify_monthly_listeners"].notna()
+        if "spotify_updated_at" in out.columns:
+            sp_fresh = out["spotify_updated_at"].apply(
+                lambda x: self._is_signal_fresh(x, max_days=7.0, reference_time=ref_time)
+            )
+        else:
+            sp_fresh = pd.Series(False, index=out.index)
+
+        for idx, (val, fresh) in enumerate(zip(has_sp_val, sp_fresh)):
+            if val and not fresh:
+                exclusions[idx].append("STALE_OR_MISSING_SPOTIFY_TIMESTAMP")
+        valid_sp = has_sp_val & sp_fresh
+
+        # 3. Instagram Validation (Followers present + Approved Status + Freshness <= 14 days)
+        has_ig_val = out["instagram_followers"].notna()
+        verified_statuses = {"AUTOMATED_VERIFIED", "MANUAL_VERIFIED"}
+
+        if "instagram_verification_status" in out.columns:
+            ig_verified = out["instagram_verification_status"].isin(verified_statuses)
+        else:
+            ig_verified = pd.Series(False, index=out.index)
+
+        for idx, (val, ver) in enumerate(zip(has_ig_val, ig_verified)):
+            if val and not ver:
+                exclusions[idx].append("UNVERIFIED_INSTAGRAM_IDENTITY")
+
+        if "instagram_updated_at" in out.columns:
+            ig_fresh = out["instagram_updated_at"].apply(
+                lambda x: self._is_signal_fresh(x, max_days=14.0, reference_time=ref_time)
+            )
+        else:
+            ig_fresh = pd.Series(False, index=out.index)
+
+        for idx, (val, ver, fresh) in enumerate(zip(has_ig_val, ig_verified, ig_fresh)):
+            if val and ver and not fresh:
+                exclusions[idx].append("STALE_OR_MISSING_INSTAGRAM_TIMESTAMP")
+
+        valid_ig = has_ig_val & ig_verified & ig_fresh
 
         out["platform_evidence_tier"] = (
-            has_yt.astype(int) + has_sp.astype(int) + has_ig.astype(int)
+            valid_yt.astype(int) + valid_sp.astype(int) + valid_ig.astype(int)
         )
+        out["evidence_exclusions"] = [";".join(e) if e else "NONE" for e in exclusions]
 
-        # 2. Sort pool deterministically by raw composite score
+        # 4. Deterministic sorting by raw composite score
         sort_cols = ["hiffi_xp_score", "production_hiffi_score", "spotify_monthly_listeners"]
         present_sort_cols = [c for c in sort_cols if c in out.columns]
         pool = out.sort_values(
@@ -277,7 +391,7 @@ class HIFFIRankingEngine:
         assigned_indices = []
         shortfall_audit = {"top_10_shortfall": False, "top_50_shortfall": False}
 
-        # Pass 1: Top 10 from Tier 3
+        # Pass 1: Select Top 10 from Tier 3 candidates
         tier3_candidates = pool[pool["platform_evidence_tier"] == 3]
         top_10_indices = tier3_candidates.head(10).index.tolist()
         assigned_indices.extend(top_10_indices)
@@ -289,13 +403,12 @@ class HIFFIRankingEngine:
             backfill = remaining_pool[remaining_pool["platform_evidence_tier"] >= 2].head(shortfall)
             assigned_indices.extend(backfill.index.tolist())
 
-            # Complete degraded fallback if Tier 2+ is also exhausted
             if len(assigned_indices) < min(10, len(pool)):
                 remaining_pool = pool.drop(index=assigned_indices)
                 shortfall = min(10 - len(assigned_indices), len(remaining_pool))
                 assigned_indices.extend(remaining_pool.head(shortfall).index.tolist())
 
-        # Pass 2: Positions 11 to 50 from Tier >= 2
+        # Pass 2: Select Positions 11 to 50 from Tier >= 2 candidates
         remaining_pool = pool.drop(index=assigned_indices)
         needed_for_50 = min(40, len(pool) - len(assigned_indices))
         tier2_plus_candidates = remaining_pool[remaining_pool["platform_evidence_tier"] >= 2]
@@ -308,11 +421,11 @@ class HIFFIRankingEngine:
             shortfall = min(50 - len(assigned_indices), len(remaining_pool))
             assigned_indices.extend(remaining_pool.head(shortfall).index.tolist())
 
-        # Pass 3: Positions 51+ with remaining artists by score order
+        # Pass 3: Positions 51+ with all remaining artists strictly by score order
         final_remaining = pool.drop(index=assigned_indices)
         assigned_indices.extend(final_remaining.index.tolist())
 
-        # 3. Assign contiguous published ranks
+        # 5. Assign consecutive published ranks
         out["hiffi_xp_published_rank"] = pd.NA
         rank_series = pd.Series(
             range(1, len(assigned_indices) + 1),