hiffi-artist-intelligence/pipeline/batch_500_runner.py

"""
pipeline/batch_500_runner.py
HIFFI Multi-Platform Batch Runner (500-Artist Cohort).
Integrates YouTube baseline, Kworb Spotify streams & velocity (+/-), and Instagram metrics.
Enforces Bayesian neutral-anchor dual-track scoring and zero-imputation governance.
"""

import json
import sys
from pathlib import Path

import numpy as np
import pandas as pd

PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from collectors.kworb_spotify_collector import KworbStreamingCollector
from core.health_detector import ScraperCircuitBreaker

# Candidate paths for base dataset in priority order
CANDIDATE_PATHS = [
    PROJECT_ROOT / "data" / "hiffi_cohort_500.csv",
    PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv",
    PROJECT_ROOT / "data" / "hiphop_artists_500.csv"
]
CHECKPOINTS_PATH = PROJECT_ROOT / "data" / "checkpoints" / "multiplatform_checkpoints.csv"
OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json"
CSV_EXPORT_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"


def get_base_dataset_path() -> Path:
    """Finds the existing base dataset across standard project names."""
    for path in CANDIDATE_PATHS:
        if path.exists():
            return path
    raise FileNotFoundError(
        f"Base dataset not found. Checked: {[str(p) for p in CANDIDATE_PATHS]}"
    )


def min_max_norm(series: pd.Series, neutral_val: float) -> pd.Series:
    """Normalizes a metric to a 0-100 scale, falling back to a neutral value for NaNs."""
    valid = series.dropna()
    if valid.empty or valid.max() == valid.min():
        return pd.Series(neutral_val, index=series.index)
    norm = (series - valid.min()) / (valid.max() - valid.min()) * 100.0
    return norm.fillna(neutral_val)


def main():
    print("=" * 90)
    print("HIFFI MULTI-PLATFORM RUNNER (100% RELIABILITY & VERIFIED GOVERNANCE)")
    print("Total Base Cohort: 500 artists")
    print("=" * 90)

    # 1. Resolve and Load Base Dataset
    base_path = get_base_dataset_path()
    print(f"[Loader] Loading base cohort from: {base_path.name}")
    df = pd.read_csv(base_path)

    # 2. Merge Checkpoints and Overrides
    if CHECKPOINTS_PATH.exists():
        chk = pd.read_csv(CHECKPOINTS_PATH)
        merge_cols = [c for c in ["name", "instagram_followers", "instagram_handle"] if c in chk.columns]
        df = df.drop(columns=[c for c in ["instagram_followers", "instagram_handle"] if c in df.columns], errors="ignore")
        df = pd.merge(df, chk[merge_cols], on="name", how="left")

    if OVERRIDES_PATH.exists():
        with open(OVERRIDES_PATH, "r") as f:
            overrides = json.load(f)
        for artist, data in overrides.items():
            mask = df["name"].str.strip().str.lower() == artist.strip().lower()
            if mask.any():
                if "instagram_handle" in data:
                    df.loc[mask, "instagram_handle"] = data["instagram_handle"]
                if "instagram_followers" in data:
                    df.loc[mask, "instagram_followers"] = data["instagram_followers"]

    # Quarantine suspicious handles with < 10,000 followers to prevent dormant penalties
    df.loc[df["instagram_followers"] < 10000, "instagram_followers"] = np.nan

    # 3. Ingest Kworb Public Streaming Metrics & Daily Velocity
    print("\n" + "=" * 90)
    print("PHASE 2: MERGING SIGNALS & ENFORCING RELIABILITY CRITERIA")
    print("=" * 90)

    kworb = KworbStreamingCollector()
    kworb.load_aggregator_tables(max_pages=2)

    df["spotify_monthly_listeners"] = df["name"].apply(lambda n: kworb.get_monthly_listeners(n))
    df["spotify_daily_change"] = df["name"].apply(lambda n: kworb.get_daily_change(n))

    # Audit Provenance Tracking
    df["youtube_data_source"] = "youtube_data_api_v3"
    df["spotify_data_source"] = df["spotify_monthly_listeners"].apply(
        lambda x: "kworb_public_aggregator" if pd.notna(x) else None
    )
    df["spotify_status"] = df["spotify_monthly_listeners"].apply(
        lambda x: "VERIFIED_PUBLIC_AGGREGATOR" if pd.notna(x) else "UNAVAILABLE_IN_TOP_5000"
    )
    df["instagram_data_source"] = df["instagram_followers"].apply(
        lambda x: "playwright_meta_scrape" if pd.notna(x) else None
    )

    # Completeness Flag
    df["has_complete_multiplatform_signals"] = (
        df["production_hiffi_score"].notna() &
        df["spotify_monthly_listeners"].notna() &
        df["instagram_followers"].notna()
    )

    # -------------------------------------------------------------
    # DUAL-TRACK SCORING WITH VELOCITY ENHANCEMENT
    # -------------------------------------------------------------
    # Production baseline is strictly preserved
    if "production_hiffi_score" not in df.columns:
        df["production_hiffi_score"] = 50.0
    if "production_hiffi_rank" not in df.columns:
        df["production_hiffi_rank"] = df["production_hiffi_score"].rank(ascending=False, method="min").astype(int)

    # Normalized Component Scores with Bayesian Neutral Anchors
    yt_norm = min_max_norm(df["production_hiffi_score"], neutral_val=50.0)

    spotify_valid = df["spotify_monthly_listeners"].dropna()
    spotify_median = (spotify_valid.median() - spotify_valid.min()) / (spotify_valid.max() - spotify_valid.min()) * 100.0 if not spotify_valid.empty else 50.0
    spotify_norm = min_max_norm(df["spotify_monthly_listeners"], neutral_val=spotify_median)

    ig_valid = df["instagram_followers"].dropna()
    ig_median = (ig_valid.median() - ig_valid.min()) / (ig_valid.max() - ig_valid.min()) * 100.0 if not ig_valid.empty else 50.0
    ig_norm = min_max_norm(df["instagram_followers"], neutral_val=ig_median)

    # Daily Velocity Calculation: Ratio of Daily Change to Total Listeners
    # Missing velocity data defaults to a neutral 0.0 momentum adjustment (no synthetic penalties)
    velocity_ratio = df["spotify_daily_change"] / df["spotify_monthly_listeners"]
    valid_velocity = velocity_ratio.dropna()

    if not valid_velocity.empty and valid_velocity.std() > 0:
        norm_velocity = (velocity_ratio - valid_velocity.mean()) / valid_velocity.std()
        momentum_adjustment = np.clip(norm_velocity.fillna(0.0) * 1.5, -5.0, 5.0)
    else:
        momentum_adjustment = 0.0

    # Composite HIFFI-XP Score (Bounded strictly between 0.0 and 100.0)
    raw_xp = (0.45 * yt_norm) + (0.35 * spotify_norm) + (0.20 * ig_norm) + momentum_adjustment
    df["hiffi_xp_score"] = np.clip(raw_xp, 0.0, 100.0).round(2)
    df["hiffi_xp_rank"] = df["hiffi_xp_score"].rank(ascending=False, method="min").astype(int)

    # 4. Export Final Audited Dataset
    CSV_EXPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
    df.to_csv(CSV_EXPORT_PATH, index=False)
    print(f"\nFinal Hardened Multi-Platform Dataset Exported: {CSV_EXPORT_PATH}")

    # 5. Pipeline Data Reliability & Audit Summary
    verified_spotify = df["spotify_monthly_listeners"].notna().sum()
    verified_ig = (df["instagram_followers"] >= 10000).sum()
    complete_signals = df["has_complete_multiplatform_signals"].sum()

    print("\n" + "=" * 90)
    print("PIPELINE DATA RELIABILITY & AUDIT SUMMARY")
    print("=" * 90)
    print(f"Total Cohort Rows:                 {len(df)}")
    print(f"Spotify Verified Telemetry Ingest: {verified_spotify} (Source: kworb_public_aggregator)")
    print(f"Verified Instagram Reach (>10k):   {verified_ig} / {len(df)}")
    print(f"Complete 3-Platform Signals:       {complete_signals} / {len(df)}")
    print("-" * 90)


if __name__ == "__main__":
    main()