""" pipeline/batch_500_runner.py HIFFI Multi-Platform Batch Runner (500-Artist Cohort). Integrates YouTube baseline, Kworb Spotify telemetry, Spotify Embed Fallback, and Multi-Signal Instagram Candidate Verification. Enforces strict dual-track scoring and zero-imputation governance. """ from datetime import datetime, timezone import json import re import sys from pathlib import Path import numpy as np import pandas as pd PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) from collectors.kworb_spotify_collector import KworbStreamingCollector from collectors.spotify_fallback_collector import SpotifyFallbackCollector from collectors.instagram_verifier import InstagramCandidateVerifier from core.health_detector import ScraperCircuitBreaker CHECKPOINTS_PATH = PROJECT_ROOT / "data" / "checkpoints" / "multiplatform_checkpoints.csv" OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json" CSV_EXPORT_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv" def clean_norm_key(text: object) -> str: """Strips non-alphanumeric characters for fuzzy-safe key matching.""" if text is None or pd.isna(text): return "" return re.sub(r"[^a-zA-Z0-9]", "", str(text)).lower() def min_max_norm(series: pd.Series, neutral_val: float) -> pd.Series: """Normalizes a metric to a 0-100 scale, falling back to a neutral value for NaNs.""" valid = series.dropna() if valid.empty or valid.max() == valid.min(): return pd.Series(neutral_val, index=series.index) norm = (series - valid.min()) / (valid.max() - valid.min()) * 100.0 return norm.fillna(neutral_val) def main(): print("=" * 90) print("HIFFI MULTI-PLATFORM RUNNER (VERIFIED SIGNALS & STRICT GOVERNANCE)") print("Total Base Cohort: 500 artists") print("=" * 90) # 1. Load Base Cohort if not CSV_EXPORT_PATH.exists(): raise FileNotFoundError(f"Base dataset missing at {CSV_EXPORT_PATH}") print(f"[Loader] Loading base cohort from: {CSV_EXPORT_PATH.name}") df = pd.read_csv(CSV_EXPORT_PATH) df = df.drop_duplicates(subset=["name"]).reset_index(drop=True) # 2. Build Robust Checkpoint Index from multiplatform_checkpoints.csv (341 records) chk_by_name = {} chk_by_handle = {} if CHECKPOINTS_PATH.exists(): chk = pd.read_csv(CHECKPOINTS_PATH) for _, r in chk.iterrows(): raw_name = str(r.get("name", "")).strip() norm_name = clean_norm_key(raw_name) raw_handle = str(r.get("instagram_handle", "")).replace("@", "").strip() norm_handle = clean_norm_key(raw_handle) f_val = r.get("instagram_followers") parsed_followers = None if pd.notna(f_val): try: f_num = float(f_val) if not np.isnan(f_num) and f_num > 0: parsed_followers = int(f_num) except (ValueError, TypeError): pass entry = { "handle": raw_handle if raw_handle and raw_handle.lower() not in ("nan", "none") else None, "followers": parsed_followers } if norm_name: chk_by_name[norm_name] = entry if norm_handle: chk_by_handle[norm_handle] = entry print(f"[Loader] Loaded {len(chk)} checkpoints ({sum(1 for v in chk_by_name.values() if v['followers'] is not None)} verified follower records).") # 3. Instagram Candidate Verification Engine print("\n[Phase 1] Executing Automated Instagram Verification & Provenance...") verifier = InstagramCandidateVerifier(overrides_path=OVERRIDES_PATH) verified_handles = [] verified_followers = [] confidence_scores = [] verification_statuses = [] audit_evidences = [] for _, row in df.iterrows(): a_name = str(row["name"]).strip() norm_a_name = clean_norm_key(a_name) # Retrieve candidate handle and followers from checkpoints first matched_chk = chk_by_name.get(norm_a_name) candidate_handle = None candidate_followers = None if matched_chk: candidate_handle = matched_chk["handle"] candidate_followers = matched_chk["followers"] # Fallback via handle normalization if not candidate_handle: row_h = row.get("instagram_handle") if pd.notna(row_h) and str(row_h).strip().lower() not in ("nan", "none", ""): candidate_handle = str(row_h).replace("@", "").strip() norm_row_h = clean_norm_key(candidate_handle) if norm_row_h in chk_by_handle and not candidate_followers: candidate_followers = chk_by_handle[norm_row_h]["followers"] # Fallback via existing row followers if candidate_followers is None: r_f = row.get("instagram_followers") if pd.notna(r_f): try: f_num = float(r_f) if not np.isnan(f_num) and f_num > 0: candidate_followers = int(f_num) except (ValueError, TypeError): pass res = verifier.evaluate_candidate( artist_name=a_name, candidate_handle=candidate_handle, follower_count=candidate_followers, youtube_channel_name=row.get("channel_title") if "channel_title" in row else None ) verified_handles.append(res["handle"]) verified_followers.append(res["followers"]) confidence_scores.append(res["confidence_score"]) verification_statuses.append(res["verification_status"]) audit_evidences.append(res["evidence"]) df["instagram_handle"] = verified_handles df["instagram_followers"] = pd.to_numeric(verified_followers, errors="coerce") df["instagram_confidence_score"] = confidence_scores df["instagram_verification_status"] = verification_statuses df["instagram_audit_evidence"] = audit_evidences df["instagram_data_source"] = df["instagram_followers"].apply( lambda x: "playwright_meta_scrape" if pd.notna(x) else None ) # 4. Spotify Multi-Tier Telemetry Ingestion print("\n[Phase 2] Ingesting Spotify Telemetry (Tier 1: Kworb Aggregator -> Tier 2: Embed Fallback)...") kworb = KworbStreamingCollector() kworb.load_aggregator_tables(max_pages=2) fallback_collector = SpotifyFallbackCollector() sp_listeners = [] sp_daily = [] sp_source = [] sp_status = [] sp_dates = [] now_utc = datetime.now(timezone.utc).isoformat() for _, row in df.iterrows(): name = str(row["name"]).strip() listeners, daily = kworb.get_artist_metrics(name) if listeners is not None: sp_listeners.append(listeners) sp_daily.append(daily) sp_source.append("kworb_public_aggregator") sp_status.append("VERIFIED_PUBLIC_AGGREGATOR") sp_dates.append(now_utc) else: sp_id = row.get("spotify_id") or row.get("spotify_uri") if pd.notna(sp_id) and str(sp_id).strip().lower() not in ("nan", "none", ""): fb_res = fallback_collector.fetch_artist_fallback(name, str(sp_id)) sp_listeners.append(fb_res["spotify_monthly_listeners"]) sp_daily.append(fb_res["spotify_daily_change"]) sp_source.append(fb_res["spotify_data_source"]) sp_status.append(fb_res["spotify_status"]) sp_dates.append(fb_res["collection_date"]) else: sp_listeners.append(None) sp_daily.append(None) sp_source.append(None) sp_status.append("UNAVAILABLE_IN_TOP_5000") sp_dates.append(now_utc) df["spotify_monthly_listeners"] = pd.to_numeric(sp_listeners, errors="coerce") df["spotify_daily_change"] = pd.to_numeric(sp_daily, errors="coerce") df["spotify_data_source"] = sp_source df["spotify_status"] = sp_status df["spotify_collection_date"] = sp_dates df["youtube_data_source"] = "youtube_data_api_v3" df["has_complete_multiplatform_signals"] = ( df["production_hiffi_score"].notna() & df["spotify_monthly_listeners"].notna() & df["instagram_followers"].notna() ) # 5. Dual-Track Scoring: Protect YouTube Baseline if "production_hiffi_score" not in df.columns: df["production_hiffi_score"] = 50.0 if "production_hiffi_rank" not in df.columns: df["production_hiffi_rank"] = df["production_hiffi_score"].rank(ascending=False, method="min").astype(int) # Calculate Experimental HIFFI-XP with Bayesian Median Neutral Anchors yt_norm = min_max_norm(df["production_hiffi_score"], neutral_val=50.0) sp_valid = df["spotify_monthly_listeners"].dropna() sp_median = (sp_valid.median() - sp_valid.min()) / (sp_valid.max() - sp_valid.min()) * 100.0 if not sp_valid.empty else 50.0 sp_norm = min_max_norm(df["spotify_monthly_listeners"], neutral_val=sp_median) ig_valid = df["instagram_followers"].dropna() ig_median = (ig_valid.median() - ig_valid.min()) / (ig_valid.max() - ig_valid.min()) * 100.0 if not ig_valid.empty else 50.0 ig_norm = min_max_norm(df["instagram_followers"], neutral_val=ig_median) # Normalized velocity modifier (bounded [-5.0, +5.0]) velocity_ratio = df["spotify_daily_change"] / df["spotify_monthly_listeners"] valid_vel = velocity_ratio.dropna() if not valid_vel.empty and valid_vel.std() > 0: norm_vel = (velocity_ratio - valid_vel.mean()) / valid_vel.std() momentum = np.clip(norm_vel.fillna(0.0) * 1.5, -5.0, 5.0) else: momentum = 0.0 raw_xp = (0.45 * yt_norm) + (0.35 * sp_norm) + (0.20 * ig_norm) + momentum df["hiffi_xp_score"] = np.clip(raw_xp, 0.0, 100.0).round(2) df["hiffi_xp_rank"] = df["hiffi_xp_score"].rank(ascending=False, method="min").astype(int) # 6. Export Audited Dataset CSV_EXPORT_PATH.parent.mkdir(parents=True, exist_ok=True) df.to_csv(CSV_EXPORT_PATH, index=False) print(f"\n[Exporter] Final Audited Dataset Exported: {CSV_EXPORT_PATH}") # Summary v_spotify = df["spotify_monthly_listeners"].notna().sum() v_ig = (df["instagram_verification_status"].isin(["MANUAL_VERIFIED", "AUTOMATED_VERIFIED"])).sum() print("\n" + "=" * 90) print("AUDIT SUMMARY (ZERO-IMPUTATION & VERIFIED COVERAGE)") print("=" * 90) print(f"Total Cohort Size: {len(df)}") print(f"Verified Spotify Telemetry: {v_spotify} / {len(df)}") print(f"Verified Instagram Profiles: {v_ig} / {len(df)}") print(f"Complete 3-Platform Coverage: {df['has_complete_multiplatform_signals'].sum()} / {len(df)}") print("=" * 90) if __name__ == "__main__": main()