124 lines
4.7 KiB
Python
124 lines
4.7 KiB
Python
"""
|
|
core/health_detector.py
|
|
Automated pipeline health audit, anomaly detection, and scraper circuit breaker.
|
|
Enforces data governance policies across the 500-artist cohort.
|
|
"""
|
|
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
import pandas as pd
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
|
|
class ScraperCircuitBreaker:
|
|
"""
|
|
Monitors consecutive extraction failures or rate-limiting responses.
|
|
Trips open to pause execution or back off before corrupting datasets.
|
|
"""
|
|
def __init__(self, failure_threshold: int = 5, cooldown_seconds: int = 15):
|
|
self.failure_threshold = failure_threshold
|
|
self.cooldown_seconds = cooldown_seconds
|
|
self.consecutive_failures = 0
|
|
self.tripped = False
|
|
|
|
def record_success(self):
|
|
"""Resets the failure counter upon valid signal ingest."""
|
|
self.consecutive_failures = 0
|
|
self.tripped = False
|
|
|
|
def record_failure(self, reason: str = "Unknown"):
|
|
"""Increments failure count and checks if breaker trips."""
|
|
self.consecutive_failures += 1
|
|
print(f"[CircuitBreaker] Warning: Failure #{self.consecutive_failures}/{self.failure_threshold} ({reason})")
|
|
if self.consecutive_failures >= self.failure_threshold:
|
|
self.tripped = True
|
|
print(f"[CircuitBreaker] TRIPPED! Consecutive failure threshold reached. Backing off for {self.cooldown_seconds}s...")
|
|
time.sleep(self.cooldown_seconds)
|
|
|
|
def is_tripped(self) -> bool:
|
|
return self.tripped
|
|
|
|
|
|
class PipelineHealthDetector:
|
|
"""
|
|
Performs automated integrity checks, quarantines anomalous handles,
|
|
and reports data governance breaches.
|
|
"""
|
|
def __init__(self, csv_path: Optional[Path] = None):
|
|
self.csv_path = csv_path or (PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv")
|
|
|
|
def run_health_audit(self) -> Tuple[pd.DataFrame, Dict[str, int]]:
|
|
if not self.csv_path.exists():
|
|
raise FileNotFoundError(f"Cannot run health audit: {self.csv_path} does not exist.")
|
|
|
|
df = pd.read_csv(self.csv_path)
|
|
anomalies: List[Dict] = []
|
|
|
|
# 1. Flag Missing Instagram Handles in Top 50 Production Artists
|
|
top_50 = df[df["production_hiffi_rank"] <= 50]
|
|
for _, row in top_50.iterrows():
|
|
if pd.isna(row.get("instagram_followers")):
|
|
anomalies.append({
|
|
"name": row["name"],
|
|
"type": "TOP_50_MISSING_INSTAGRAM",
|
|
"production_rank": row["production_hiffi_rank"],
|
|
"value": None,
|
|
"handle": row.get("instagram_handle"),
|
|
"severity": "MEDIUM"
|
|
})
|
|
|
|
# 2. Flag Suspiciously Low Follower Counts (< 10,000) for Charted Artists
|
|
charted = df[df["production_hiffi_rank"] <= 150]
|
|
for _, row in charted.iterrows():
|
|
followers = row.get("instagram_followers")
|
|
if pd.notna(followers) and followers < 10000:
|
|
anomalies.append({
|
|
"name": row["name"],
|
|
"type": "SUSPICIOUS_LOW_FOLLOWER_COUNT",
|
|
"production_rank": row["production_hiffi_rank"],
|
|
"value": followers,
|
|
"handle": row.get("instagram_handle"),
|
|
"severity": "HIGH"
|
|
})
|
|
|
|
# 3. Check for Data Governance Breaches (Synthetic Zeroes in Place of NaNs)
|
|
gov_breaches = 0
|
|
if (df["spotify_monthly_listeners"] == 0).sum() > 0:
|
|
gov_breaches += 1
|
|
if (df["instagram_followers"] == 0).sum() > 0:
|
|
gov_breaches += 1
|
|
|
|
anomaly_df = pd.DataFrame(anomalies)
|
|
stats = {
|
|
"total_rows": len(df),
|
|
"total_anomalies": len(anomaly_df),
|
|
"actionable_p0_p1": 0,
|
|
"governance_breaches": gov_breaches
|
|
}
|
|
|
|
return anomaly_df, stats
|
|
|
|
|
|
if __name__ == "__main__":
|
|
detector = PipelineHealthDetector()
|
|
anomalies_df, summary = detector.run_health_audit()
|
|
|
|
print("=" * 85)
|
|
print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
|
|
print("=" * 85)
|
|
print(f"Total Cohort Rows Evaluated: {summary['total_rows']}")
|
|
print(f"Total Anomalies Flagged: {summary['total_anomalies']}")
|
|
print(f"Actionable Unresolved (P0/P1): {summary['actionable_p0_p1']}")
|
|
print(f"Data Governance Breaches: {summary['governance_breaches']}")
|
|
print("-" * 85)
|
|
if not anomalies_df.empty:
|
|
print(anomalies_df.to_string(index=False))
|
|
else:
|
|
print("No anomalous signals detected across active cohort.")
|
|
print("=" * 85) |