hiffi-artist-intelligence/core/health_detector.py

"""
core/health_detector.py
Automated Anomaly Detection & Data Health Diagnostics for HIFFI Pipeline.

Evaluates:
- High-velocity artists mapped to low-follower or inactive social profiles.
- Unextracted zero-follower anomalies (e.g. rate-limit or UI layout shifts).
- Missing telemetry on Top 50 cohort.
- Strict streaming governance (valid source attribution, zero-imputation bans).
- Auto-matching against data/handle_overrides.json.
"""

import sys
import json
from pathlib import Path
from typing import Dict, List, Any

PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

import pandas as pd
import numpy as np


class PipelineHealthDetector:
    def __init__(
        self,
        dataset_path: Path = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv",
        overrides_path: Path = PROJECT_ROOT / "data" / "handle_overrides.json"
    ):
        self.dataset_path = Path(dataset_path)
        self.overrides_path = Path(overrides_path)
        self.overrides = self._load_overrides()

    def _load_overrides(self) -> Dict[str, str]:
        if self.overrides_path.exists():
            try:
                with open(self.overrides_path, "r", encoding="utf-8") as f:
                    return json.load(f)
            except Exception:
                return {}
        return {}

    def audit_dataset(self, df: pd.DataFrame = None) -> Dict[str, Any]:
        """
        Executes an automated multi-point anomaly & governance check.
        """
        if df is None:
            if not self.dataset_path.exists():
                raise FileNotFoundError(f"Dataset not found at {self.dataset_path}")
            df = pd.read_csv(self.dataset_path)

        anomalies = []

        # 1. Suspicious Low Follower Anomaly (< 10,000 followers for Top 150 production artists)
        suspicious_low = df[
            (df["production_hiffi_rank"] <= 150) &
            (df["instagram_followers"].notna()) &
            (df["instagram_followers"] < 10000)
        ]
        for _, row in suspicious_low.iterrows():
            artist_name = row["name"]
            is_overridden = artist_name in self.overrides
            anomalies.append({
                "name": artist_name,
                "type": "SUSPICIOUS_LOW_FOLLOWER_COUNT",
                "production_rank": row["production_hiffi_rank"],
                "value": row["instagram_followers"],
                "handle": row.get("instagram_handle", "N/A"),
                "has_override": is_overridden,
                "severity": "HIGH" if not is_overridden else "RESOLVED_BY_OVERRIDE"
            })

        # 2. Zero-Follower Ingestion Bug (Instagram followers == 0)
        zero_followers = df[
            (df["instagram_followers"].notna()) &
            (df["instagram_followers"] == 0)
        ]
        for _, row in zero_followers.iterrows():
            artist_name = row["name"]
            is_overridden = artist_name in self.overrides
            anomalies.append({
                "name": artist_name,
                "type": "ZERO_FOLLOWER_EXTRACTION_FAILURE",
                "production_rank": row.get("production_hiffi_rank", np.nan),
                "value": 0,
                "handle": row.get("instagram_handle", "N/A"),
                "has_override": is_overridden,
                "severity": "CRITICAL" if not is_overridden else "RESOLVED_BY_OVERRIDE"
            })

        # 3. Missing Instagram Telemetry in Core Top 50
        top_50_missing_ig = df[
            (df["production_hiffi_rank"] <= 50) &
            (df["instagram_followers"].isna())
        ]
        for _, row in top_50_missing_ig.iterrows():
            artist_name = row["name"]
            anomalies.append({
                "name": artist_name,
                "type": "TOP_50_MISSING_INSTAGRAM",
                "production_rank": row["production_hiffi_rank"],
                "value": "NaN",
                "handle": row.get("instagram_handle", "N/A"),
                "has_override": artist_name in self.overrides,
                "severity": "MEDIUM"
            })

        # 4. Strict Streaming Governance Violations
        # A. Unattributed populated Spotify listeners (source must be kworb_public_aggregator)
        unattributed_spotify = df[
            (df["spotify_monthly_listeners"].notna()) &
            (df["spotify_data_source"] != "kworb_public_aggregator")
        ]
        for _, row in unattributed_spotify.iterrows():
            anomalies.append({
                "name": row["name"],
                "type": "UNATTRIBUTED_STREAMING_DATA",
                "production_rank": row.get("production_hiffi_rank", np.nan),
                "value": row["spotify_monthly_listeners"],
                "handle": "N/A",
                "has_override": False,
                "severity": "FATAL_GOVERNANCE_BREACH"
            })

        # B. Zero-imputed streaming metrics (Listeners == 0)
        zero_listeners = df[
            (df["spotify_monthly_listeners"].notna()) &
            (df["spotify_monthly_listeners"] <= 0)
        ]
        for _, row in zero_listeners.iterrows():
            anomalies.append({
                "name": row["name"],
                "type": "ZERO_IMPUTED_STREAMING_METRIC",
                "production_rank": row.get("production_hiffi_rank", np.nan),
                "value": row["spotify_monthly_listeners"],
                "handle": "N/A",
                "has_override": False,
                "severity": "FATAL_GOVERNANCE_BREACH"
            })

        df_anomalies = pd.DataFrame(anomalies)

        summary = {
            "total_records_evaluated": len(df),
            "total_anomalies_flagged": len(anomalies),
            "actionable_unresolved_count": len(
                df_anomalies[df_anomalies["severity"].isin(["CRITICAL", "HIGH", "FATAL_GOVERNANCE_BREACH"])]
            ) if not df_anomalies.empty else 0,
            "governance_violations": len(unattributed_spotify) + len(zero_listeners),
            "anomalies": anomalies
        }
        return summary

    def print_diagnostic_report(self):
        """Prints diagnostic report to console."""
        print("=" * 85)
        print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
        print("=" * 85)

        summary = self.audit_dataset()
        print(f"Total Cohort Rows Evaluated:   {summary['total_records_evaluated']}")
        print(f"Total Anomalies Flagged:       {summary['total_anomalies_flagged']}")
        print(f"Actionable Unresolved (P0/P1): {summary['actionable_unresolved_count']}")
        print(f"Data Governance Breaches:      {summary['governance_violations']}")
        print("-" * 85)

        if summary["total_anomalies_flagged"] > 0:
            df_anomalies = pd.DataFrame(summary["anomalies"])
            cols = ["name", "type", "production_rank", "value", "handle", "severity"]
            print(df_anomalies[cols].to_string(index=False))
        else:
            print("Status: HEALTHY. Zero anomalies or governance breaches detected.")

        print("=" * 85)
        return summary


if __name__ == "__main__":
    detector = PipelineHealthDetector()
    detector.print_diagnostic_report()