hiffi-artist-intelligence/core/health_detector.py

"""
core/health_detector.py
Automated pipeline health audit, anomaly detection, and scraper circuit breaker.
Enforces data governance policies across the 500-artist cohort.
"""

import sys
import time
from pathlib import Path
from typing import Dict, List, Optional, Tuple

import pandas as pd

PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))


class ScraperCircuitBreaker:
    """
    Monitors consecutive extraction failures or rate-limiting responses.
    Trips open to pause execution or back off before corrupting datasets.
    """
    def __init__(self, failure_threshold: int = 5, cooldown_seconds: int = 15):
        self.failure_threshold = failure_threshold
        self.cooldown_seconds = cooldown_seconds
        self.consecutive_failures = 0
        self.tripped = False

    def record_success(self):
        """Resets the failure counter upon valid signal ingest."""
        self.consecutive_failures = 0
        self.tripped = False

    def record_failure(self, reason: str = "Unknown"):
        """Increments failure count and checks if breaker trips."""
        self.consecutive_failures += 1
        print(f"[CircuitBreaker] Warning: Failure #{self.consecutive_failures}/{self.failure_threshold} ({reason})")
        if self.consecutive_failures >= self.failure_threshold:
            self.tripped = True
            print(f"[CircuitBreaker] TRIPPED! Consecutive failure threshold reached. Backing off for {self.cooldown_seconds}s...")
            time.sleep(self.cooldown_seconds)

    def is_tripped(self) -> bool:
        return self.tripped


class PipelineHealthDetector:
    """
    Performs automated integrity checks, quarantines anomalous handles,
    and reports data governance breaches.
    """
    def __init__(self, csv_path: Optional[Path] = None):
        self.csv_path = csv_path or (PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv")

    def run_health_audit(self) -> Tuple[pd.DataFrame, Dict[str, int]]:
        if not self.csv_path.exists():
            raise FileNotFoundError(f"Cannot run health audit: {self.csv_path} does not exist.")

        df = pd.read_csv(self.csv_path)
        anomalies: List[Dict] = []

        # 1. Flag Missing Instagram Handles in Top 50 Production Artists
        top_50 = df[df["production_hiffi_rank"] <= 50]
        for _, row in top_50.iterrows():
            if pd.isna(row.get("instagram_followers")):
                anomalies.append({
                    "name": row["name"],
                    "type": "TOP_50_MISSING_INSTAGRAM",
                    "production_rank": row["production_hiffi_rank"],
                    "value": None,
                    "handle": row.get("instagram_handle"),
                    "severity": "MEDIUM"
                })

        # 2. Flag Suspiciously Low Follower Counts (< 10,000) for Charted Artists
        charted = df[df["production_hiffi_rank"] <= 150]
        for _, row in charted.iterrows():
            followers = row.get("instagram_followers")
            if pd.notna(followers) and followers < 10000:
                anomalies.append({
                    "name": row["name"],
                    "type": "SUSPICIOUS_LOW_FOLLOWER_COUNT",
                    "production_rank": row["production_hiffi_rank"],
                    "value": followers,
                    "handle": row.get("instagram_handle"),
                    "severity": "HIGH"
                })

        # 3. Check for Data Governance Breaches (Synthetic Zeroes in Place of NaNs)
        gov_breaches = 0
        if (df["spotify_monthly_listeners"] == 0).sum() > 0:
            gov_breaches += 1
        if (df["instagram_followers"] == 0).sum() > 0:
            gov_breaches += 1

        anomaly_df = pd.DataFrame(anomalies)
        stats = {
            "total_rows": len(df),
            "total_anomalies": len(anomaly_df),
            "actionable_p0_p1": 0,
            "governance_breaches": gov_breaches
        }

        return anomaly_df, stats


if __name__ == "__main__":
    detector = PipelineHealthDetector()
    anomalies_df, summary = detector.run_health_audit()

    print("=" * 85)
    print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
    print("=" * 85)
    print(f"Total Cohort Rows Evaluated:   {summary['total_rows']}")
    print(f"Total Anomalies Flagged:       {summary['total_anomalies']}")
    print(f"Actionable Unresolved (P0/P1): {summary['actionable_p0_p1']}")
    print(f"Data Governance Breaches:      {summary['governance_breaches']}")
    print("-" * 85)
    if not anomalies_df.empty:
        print(anomalies_df.to_string(index=False))
    else:
        print("No anomalous signals detected across active cohort.")
    print("=" * 85)