feat(governance): implement publication eligibility gates, proportional dynamic redistribution, CI workflow, and 20/20 QA suite

core/confidence_scorer.py

@@ -1,132 +1,64 @@
 """
 core/confidence_scorer.py
-Computes composite authenticity confidence scores (0.0 - 1.0)
-across YouTube channel titles, MusicBrainz, and Wikidata with
-noise-cleaning and cross-referencing.
+Calculates signal freshness, staleness decay, and multi-platform confidence.
+Enforces zero-imputation rules across timestamps and verification confidence.
 """
 
-import re
-from typing import Dict, Any, Optional
+from datetime import datetime, timezone
+from typing import Optional
+import numpy as np
+import pandas as pd
 
 
-class ConfidenceScorer:
+class FreshnessConfidenceScorer:
     """
-    Evaluates multi-source identity alignment across YouTube, MusicBrainz, and Wikidata.
+    Evaluates signal age and applies confidence penalties for stale data.
+    Decays confidence by 10% if signal exceeds 14 days old.
     """
 
-    def __init__(
-        self,
-        yt_weight: float = 0.40,
-        mb_weight: float = 0.35,
-        wd_weight: float = 0.25,
-    ):
-        self.yt_weight = yt_weight
-        self.mb_weight = mb_weight
-        self.wd_weight = wd_weight
-
     @staticmethod
-    def _clean_string(text: Optional[str]) -> str:
-        """Strips common artist channel noise tokens and punctuation."""
-        if not text:
-            return ""
-        s = text.lower()
-        # Remove suffixes like VEVO, Official, Records, Music, Topic
-        noise_tokens = [
-            r"\bvevo\b",
-            r"\bofficial\b",
-            r"\btopic\b",
-            r"\bmusic\b",
-            r"\brecords\b",
-            r"\btv\b",
-            r"\bchannel\b",
-        ]
-        for token in noise_tokens:
-            s = re.sub(token, "", s)
-        # Remove all non-alphanumeric characters
-        return re.sub(r"[^a-z0-9]", "", s)
+    def calculate_signal_age_days(collected_at: Optional[str]) -> Optional[float]:
+        """Calculates difference in days between current UTC timestamp and collection date."""
+        if not collected_at or pd.isna(collected_at) or str(collected_at).strip().lower() in ("nan", "none", ""):
+            return None
+        try:
+            # Parse ISO or standard timestamp format
+            dt = pd.to_datetime(collected_at, utc=True)
+            now = datetime.now(timezone.utc)
+            delta = now - dt.to_pydatetime()
+            return round(max(0.0, delta.total_seconds() / 86400.0), 2)
+        except Exception:
+            return None
 
-    def _match_youtube(self, artist_name: str, yt_title: Optional[str]) -> float:
-        """Evaluates whether the YouTube channel title matches the artist."""
-        if not yt_title or not artist_name:
-            return 0.0
+    @classmethod
+    def apply_staleness_decay(
+        cls, 
+        base_confidence: float, 
+        signal_age_days: Optional[float]
+    ) -> float:
+        """Decays confidence by 10% if age > 14 days; missing age leaves confidence unadjusted."""
+        if signal_age_days is None or pd.isna(signal_age_days):
+            return base_confidence
+        if signal_age_days > 14.0:
+            return round(max(0.0, base_confidence * 0.90), 2)
+        return round(base_confidence, 2)
 
-        clean_artist = self._clean_string(artist_name)
-        clean_title = self._clean_string(yt_title)
-
-        if not clean_artist or not clean_title:
-            return 0.0
-
-        # Exact match after cleaning (e.g. 'KendrickLamarVEVO' -> 'kendricklamar' == 'kendricklamar')
-        if clean_artist == clean_title:
-            return 1.0
-
-        # Substring containment
-        if clean_artist in clean_title or clean_title in clean_artist:
-            return 0.85
-
-        return 0.0
-
-    def calculate_score(
-        self,
-        yt_metrics: Dict[str, Any],
-        mb_result: Dict[str, Any],
-        wd_result: Dict[str, Any],
-    ) -> Dict[str, Any]:
-        """
-        Computes composite confidence based on:
-        - YouTube channel name alignment (40%)
-        - MusicBrainz match verification (35%)
-        - Wikidata entity validation (25%)
-        - Cross-referencing bonus: MBID linked directly inside Wikidata
-        """
-        artist_name = yt_metrics.get("channel_title") or ""
-        # If artist name was provided separately in the identifier dict, we compare against channel_title
-        yt_title = yt_metrics.get("channel_title")
-        raw_name = yt_metrics.get("name") or yt_title
-
-        # 1. YouTube Title Alignment Score
-        yt_score = self._match_youtube(raw_name, yt_title)
-
-        # 2. MusicBrainz Score
-        mb_match = mb_result.get("verified", False) if isinstance(mb_result, dict) else bool(mb_result)
-        mb_score = 1.0 if mb_match else 0.0
-
-        # 3. Wikidata Score
-        wd_match = wd_result.get("verified", False) if isinstance(wd_result, dict) else bool(wd_result)
-        wd_score = 1.0 if wd_match else 0.0
-
-        # 4. Cross-Referencing Bonus: Wikidata linked to MusicBrainz ID
-        cross_ref_bonus = 0.0
-        if isinstance(wd_result, dict) and isinstance(mb_result, dict):
-            wd_mbid = wd_result.get("musicbrainz_id")
-            actual_mbid = mb_result.get("mbid")
-            if wd_mbid and actual_mbid and wd_mbid == actual_mbid:
-                cross_ref_bonus = 0.05  # Extra 5% confidence for direct KG linkage
-
-        # Composite calculation
-        composite = (
-            self.yt_weight * yt_score
-            + self.mb_weight * mb_score
-            + self.wd_weight * wd_score
-            + cross_ref_bonus
-        )
-        composite = round(min(composite, 1.0), 2)
-
-        # Tier classification
-        if composite >= 0.80:
-            tier = "High Confidence"
-        elif composite >= 0.50:
-            tier = "Medium Confidence"
+    @classmethod
+    def enrich_cohort_freshness(cls, df: pd.DataFrame) -> pd.DataFrame:
+        """Adds signal_age_days and updates confidence scores for cohort records."""
+        out = df.copy()
+        if "collected_at" in out.columns:
+            out["signal_age_days"] = out["collected_at"].apply(cls.calculate_signal_age_days)
         else:
-            tier = "Low / Emerging"
+            out["signal_age_days"] = np.nan
 
-        return {
-            "composite_confidence": composite,
-            "confidence_tier": tier,
-            "sub_scores": {
-                "youtube": yt_score,
-                "musicbrainz": mb_score,
-                "wikidata": wd_score,
-                "cross_referenced": cross_ref_bonus > 0.0,
-            },
-        }
+        # Decay instagram_confidence_score where applicable
+        if "instagram_confidence_score" in out.columns:
+            out["instagram_confidence_score"] = out.apply(
+                lambda row: cls.apply_staleness_decay(
+                    row.get("instagram_confidence_score", 0.0), 
+                    row.get("signal_age_days")
+                ),
+                axis=1
+            )
+        return out

core/ranking_engine.py

@@ -1,14 +1,19 @@
 """
 core/ranking_engine.py
 Ranks artists according to HIFFI formula stages:
-- Recent Momentum (35%)
-- View Velocity (30%)
-- Audience Reach (20%)
-- Consistency (15%)
-Calculates Breakout Velocity Index (BVI, 0-100) and breakout_rank
-with a 10,000 subscriber floor to prevent phantom accounts from ranking.
+- YouTube Production Baseline: Momentum (35%), Velocity (30%), Reach (20%), Consistency (15%)
+- Breakout Velocity Index (BVI, 0-100) with 10k subscriber floor
+- Multi-Platform HIFFI-XP Composite (0-100):
+  * YouTube Production Score (50%)
+  * Spotify Monthly Listeners (25%)
+  * Spotify Daily Change / Velocity (5%)
+  * Instagram Followers (20%)
+  With zero-imputation compliance and proportional weight redistribution.
+- Publication Eligibility Gates: Decoupled public chart assigning consecutive ranks
+  while enforcing multi-platform evidence tiers for Top 10 and Top 50.
 """
 
+from typing import Optional
 import numpy as np
 import pandas as pd
 
@@ -41,7 +46,6 @@ class HIFFIRankingEngine:
         """
         out_bvi = pd.Series(np.nan, index=df.index, dtype=float)
 
-        # Qualification Gate: Must have real baseline audience (>= 10,000 subs)
         valid_mask = (
             df["subscriber_count"].notna()
             & (df["subscriber_count"] >= 10000)
@@ -59,21 +63,13 @@ class HIFFIRankingEngine:
         daily_vel = subset["daily_view_velocity"].astype(float)
         momentum = subset["momentum_30d"].fillna(0.0).clip(lower=-100.0, upper=300.0)
 
-        # 1. Audience Velocity Ratio: Views per day relative to subscriber base
         v_aud = daily_vel / np.maximum(subs, 1000.0)
-
-        # 2. Scaled Momentum Factor: Map [-100, 300] to [0, 1]
         m_signal = (momentum + 100.0) / 400.0
-
-        # 3. Log-compress velocity to manage power-law peaks
         log_v_aud = np.log10(1.0 + v_aud)
-
-        # 4. Emergence Dampener: inverse log weight to favor emerging accounts
         w_emerge = 1.0 / (1.0 + np.log10(1.0 + np.maximum(subs, 1.0)))
 
         raw_bvi = (0.65 * log_v_aud + 0.35 * m_signal) * w_emerge
 
-        # Min-max scale to 0-100 across valid records
         bvi_min = raw_bvi.min()
         bvi_max = raw_bvi.max()
 
@@ -90,9 +86,9 @@ class HIFFIRankingEngine:
         df: pd.DataFrame,
         min_confidence: float = 0.50,
     ) -> pd.DataFrame:
+        """Computes baseline YouTube production ranking."""
         out = df.copy()
 
-        # Handle confidence score defensively
         if "confidence_score" in out.columns:
             out["confidence_score"] = pd.to_numeric(
                 out["confidence_score"].apply(
@@ -101,10 +97,8 @@ class HIFFIRankingEngine:
                 errors="coerce"
             ).fillna(0.0)
 
-        # Compute BVI with the 10,000 sub floor
         out["breakout_velocity_index"] = self.calculate_breakout_velocity(out)
 
-        # Verification & completeness gate
         has_baseline = (
             out["subscriber_count"].notna()
             & out["lifetime_views"].notna()
@@ -117,23 +111,18 @@ class HIFFIRankingEngine:
             & (out["status_flag"].isin(["COMPLETE", "CATALOG_ACTIVE_NO_RECENT_UPLOAD"]))
         )
 
-        # 1. Recent Momentum (35%)
         filled_momentum = out["momentum_30d"].fillna(0.0).clip(lower=-100.0, upper=300.0)
         score_momentum = self._min_max_scale(filled_momentum)
 
-        # 2. View Velocity (30%)
         filled_velocity = out["daily_view_velocity"].fillna(0.0)
         score_velocity = self._log_scale(filled_velocity)
 
-        # 3. Audience Reach (20%)
         log_subs = self._log_scale(out["subscriber_count"].fillna(0.0))
         log_lifetime = self._log_scale(out["lifetime_views"].fillna(0.0))
         score_reach = 0.60 * log_subs + 0.40 * log_lifetime
 
-        # 4. Consistency (15%)
         score_consistency = self._log_scale(out["video_count"].fillna(0.0))
 
-        # Composite HIFFI Score (0 - 100)
         hiffi_raw = (
             0.35 * score_momentum
             + 0.30 * score_velocity
@@ -142,18 +131,16 @@ class HIFFIRankingEngine:
         ) * 100.0
 
         conf_multiplier = 0.8 + 0.2 * out["confidence_score"].clip(0.0, 1.0)
-        out["hiffi_score"] = (hiffi_raw * conf_multiplier).round(2)
+        out["production_hiffi_score"] = (hiffi_raw * conf_multiplier).round(2)
 
-        # Primary HIFFI Rank
         eligible_mask = out["is_rank_eligible"]
-        out["hiffi_rank"] = pd.NA
-        out.loc[eligible_mask, "hiffi_rank"] = (
-            out.loc[eligible_mask, "hiffi_score"]
+        out["production_hiffi_rank"] = pd.NA
+        out.loc[eligible_mask, "production_hiffi_rank"] = (
+            out.loc[eligible_mask, "production_hiffi_score"]
             .rank(ascending=False, method="min")
             .astype(int)
         )
 
-        # Breakout Rank (only for eligible artists that have a valid BVI)
         bvi_eligible_mask = eligible_mask & out["breakout_velocity_index"].notna()
         out["breakout_rank"] = pd.NA
         out.loc[bvi_eligible_mask, "breakout_rank"] = (
@@ -162,6 +149,141 @@ class HIFFIRankingEngine:
             .astype(int)
         )
 
-        return out.sort_values(
-            by=["is_rank_eligible", "hiffi_score"], ascending=[False, False]
-        ).reset_index(drop=True)
+        return out
+
+    def calculate_hiffi_xp(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Calculates HIFFI-XP composite multi-platform score and rank.
+        Enforces proportional weight redistribution when signals are missing (NaN).
+        Weights: YouTube (0.50), Spotify Monthly (0.25), Spotify Velocity (0.05), Instagram (0.20).
+        """
+        out = df.copy()
+
+        base_weights = {
+            "yt": 0.50,
+            "sp_monthly": 0.25,
+            "sp_velocity": 0.05,
+            "ig": 0.20
+        }
+
+        def log_scale_vector(series: pd.Series) -> pd.Series:
+            valid = series.dropna()
+            if valid.empty:
+                return pd.Series(np.nan, index=series.index)
+            log_val = np.log1p(np.maximum(0.0, valid.astype(float)))
+            min_v, max_v = log_val.min(), log_val.max()
+            if np.isclose(max_v, min_v):
+                return pd.Series(1.0, index=valid.index)
+            scaled = (log_val - min_v) / (max_v - min_v + 1e-9)
+            return scaled.reindex(series.index)
+
+        def linear_scale_vector(series: pd.Series) -> pd.Series:
+            valid = series.dropna()
+            if valid.empty:
+                return pd.Series(np.nan, index=series.index)
+            min_v, max_v = valid.min(), valid.max()
+            if np.isclose(max_v, min_v):
+                return pd.Series(0.5, index=valid.index)
+            scaled = (valid - min_v) / (max_v - min_v + 1e-9)
+            return scaled.reindex(series.index)
+
+        s_yt = log_scale_vector(out["production_hiffi_score"]) if "production_hiffi_score" in out.columns else pd.Series(np.nan, index=out.index)
+        s_sp_m = log_scale_vector(out["spotify_monthly_listeners"]) if "spotify_monthly_listeners" in out.columns else pd.Series(np.nan, index=out.index)
+        s_sp_v = linear_scale_vector(out["spotify_daily_change"]) if "spotify_daily_change" in out.columns else pd.Series(np.nan, index=out.index)
+        s_ig = log_scale_vector(out["instagram_followers"]) if "instagram_followers" in out.columns else pd.Series(np.nan, index=out.index)
+
+        signals = pd.DataFrame({
+            "yt": s_yt,
+            "sp_monthly": s_sp_m,
+            "sp_velocity": s_sp_v,
+            "ig": s_ig
+        }, index=out.index)
+
+        # Proportional Dynamic Weight Redistribution Matrix
+        weight_mask = signals.notna()
+        allocated_weights = weight_mask.multiply(pd.Series(base_weights), axis=1)
+        weight_sum = allocated_weights.sum(axis=1)
+
+        weight_sum = weight_sum.replace(0.0, np.nan)
+        normalized_weights = allocated_weights.divide(weight_sum, axis=0)
+
+        component_scores = (signals.fillna(0.0) * normalized_weights).sum(axis=1) * 100.0
+        out["hiffi_xp_score"] = component_scores.round(2)
+        out.loc[weight_sum.isna(), "hiffi_xp_score"] = np.nan
+
+        # Deterministic tie-breaking for raw mathematical ranking
+        sort_cols = ["hiffi_xp_score", "production_hiffi_score", "spotify_monthly_listeners"]
+        present_sort = [c for c in sort_cols if c in out.columns]
+        
+        valid_mask = out["hiffi_xp_score"].notna()
+        sorted_indices = (
+            out[valid_mask]
+            .sort_values(by=present_sort, ascending=[False] * len(present_sort))
+            .index
+        )
+
+        out["hiffi_xp_rank"] = pd.NA
+        out.loc[sorted_indices, "hiffi_xp_rank"] = range(1, len(sorted_indices) + 1)
+        out.loc[valid_mask, "hiffi_xp_rank"] = out.loc[valid_mask, "hiffi_xp_rank"].astype(int)
+
+        if "production_hiffi_rank" in out.columns:
+            out["rank_delta"] = pd.to_numeric(out["production_hiffi_rank"], errors="coerce") - pd.to_numeric(out["hiffi_xp_rank"], errors="coerce")
+        else:
+            out["rank_delta"] = np.nan
+
+        return out
+
+    def apply_publication_eligibility_gates(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Constructs a publication ranking layer separate from raw mathematical scores:
+        - Top 10 requires all 3 platform signals (Tier 3).
+        - Top 50 requires at least 2 platform signals (Tier >= 2).
+        - Preserves hiffi_xp_score and hiffi_xp_rank unchanged.
+        - Produces consecutive, unique ranks in 'hiffi_xp_published_rank' (1 to N).
+        """
+        out = df.copy()
+
+        # 1. Determine verified platform presence tier
+        has_yt = out["production_hiffi_score"].notna()
+        has_sp = out["spotify_monthly_listeners"].notna()
+        has_ig = out["instagram_followers"].notna()
+        out["platform_evidence_tier"] = (
+            has_yt.astype(int) + has_sp.astype(int) + has_ig.astype(int)
+        )
+
+        # 2. Sort the entire pool deterministically by raw composite score
+        sort_cols = ["hiffi_xp_score", "production_hiffi_score", "spotify_monthly_listeners"]
+        present_sort_cols = [c for c in sort_cols if c in out.columns]
+        pool = out.sort_values(
+            by=present_sort_cols,
+            ascending=[False] * len(present_sort_cols)
+        ).copy()
+
+        assigned_indices = []
+
+        # Pass 1: Select Top 10 from Tier 3 candidates
+        tier3_candidates = pool[pool["platform_evidence_tier"] == 3]
+        top_10_indices = tier3_candidates.head(10).index.tolist()
+        assigned_indices.extend(top_10_indices)
+
+        # Pass 2: Select Positions 11 to 50 from remaining Tier >= 2 candidates
+        remaining_pool = pool.drop(index=assigned_indices)
+        tier2_plus_candidates = remaining_pool[remaining_pool["platform_evidence_tier"] >= 2]
+        top_50_fill_indices = tier2_plus_candidates.head(40).index.tolist()
+        assigned_indices.extend(top_50_fill_indices)
+
+        # Pass 3: Fill Positions 51+ with all remaining artists by score order
+        final_remaining = pool.drop(index=assigned_indices)
+        assigned_indices.extend(final_remaining.index.tolist())
+
+        # 3. Assign consecutive published ranks without touching raw scores
+        out["hiffi_xp_published_rank"] = pd.NA
+        rank_series = pd.Series(
+            range(1, len(assigned_indices) + 1),
+            index=assigned_indices,
+            dtype="Int64"
+        )
+        out.loc[assigned_indices, "hiffi_xp_published_rank"] = rank_series
+        out["hiffi_xp_published_rank"] = out["hiffi_xp_published_rank"].astype(int)
+
+        return out