feat: integrate daily velocity ingestion, circuit breaker, clean requirements, and resilient batch loader
This commit is contained in:
Srilekha committed
1 parent
66db687a58
commit
90b4956635
6 files changed
+832
-948
No files matched your search
+98
-156
@@ -1,182 +1,124 @@
|
||||
"""
|
||||
core/health_detector.py
|
||||
Automated Anomaly Detection & Data Health Diagnostics for HIFFI Pipeline.
|
||||
|
||||
Evaluates:
|
||||
- High-velocity artists mapped to low-follower or inactive social profiles.
|
||||
- Unextracted zero-follower anomalies (e.g. rate-limit or UI layout shifts).
|
||||
- Missing telemetry on Top 50 cohort.
|
||||
- Strict streaming governance (valid source attribution, zero-imputation bans).
|
||||
- Auto-matching against data/handle_overrides.json.
|
||||
Automated pipeline health audit, anomaly detection, and scraper circuit breaker.
|
||||
Enforces data governance policies across the 500-artist cohort.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import pandas as pd
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
class ScraperCircuitBreaker:
|
||||
"""
|
||||
Monitors consecutive extraction failures or rate-limiting responses.
|
||||
Trips open to pause execution or back off before corrupting datasets.
|
||||
"""
|
||||
def __init__(self, failure_threshold: int = 5, cooldown_seconds: int = 15):
|
||||
self.failure_threshold = failure_threshold
|
||||
self.cooldown_seconds = cooldown_seconds
|
||||
self.consecutive_failures = 0
|
||||
self.tripped = False
|
||||
|
||||
def record_success(self):
|
||||
"""Resets the failure counter upon valid signal ingest."""
|
||||
self.consecutive_failures = 0
|
||||
self.tripped = False
|
||||
|
||||
def record_failure(self, reason: str = "Unknown"):
|
||||
"""Increments failure count and checks if breaker trips."""
|
||||
self.consecutive_failures += 1
|
||||
print(f"[CircuitBreaker] Warning: Failure #{self.consecutive_failures}/{self.failure_threshold} ({reason})")
|
||||
if self.consecutive_failures >= self.failure_threshold:
|
||||
self.tripped = True
|
||||
print(f"[CircuitBreaker] TRIPPED! Consecutive failure threshold reached. Backing off for {self.cooldown_seconds}s...")
|
||||
time.sleep(self.cooldown_seconds)
|
||||
|
||||
def is_tripped(self) -> bool:
|
||||
return self.tripped
|
||||
|
||||
|
||||
class PipelineHealthDetector:
|
||||
def __init__(
|
||||
self,
|
||||
dataset_path: Path = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv",
|
||||
overrides_path: Path = PROJECT_ROOT / "data" / "handle_overrides.json"
|
||||
):
|
||||
self.dataset_path = Path(dataset_path)
|
||||
self.overrides_path = Path(overrides_path)
|
||||
self.overrides = self._load_overrides()
|
||||
"""
|
||||
Performs automated integrity checks, quarantines anomalous handles,
|
||||
and reports data governance breaches.
|
||||
"""
|
||||
def __init__(self, csv_path: Optional[Path] = None):
|
||||
self.csv_path = csv_path or (PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv")
|
||||
|
||||
def _load_overrides(self) -> Dict[str, str]:
|
||||
if self.overrides_path.exists():
|
||||
try:
|
||||
with open(self.overrides_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
return {}
|
||||
def run_health_audit(self) -> Tuple[pd.DataFrame, Dict[str, int]]:
|
||||
if not self.csv_path.exists():
|
||||
raise FileNotFoundError(f"Cannot run health audit: {self.csv_path} does not exist.")
|
||||
|
||||
def audit_dataset(self, df: pd.DataFrame = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Executes an automated multi-point anomaly & governance check.
|
||||
"""
|
||||
if df is None:
|
||||
if not self.dataset_path.exists():
|
||||
raise FileNotFoundError(f"Dataset not found at {self.dataset_path}")
|
||||
df = pd.read_csv(self.dataset_path)
|
||||
df = pd.read_csv(self.csv_path)
|
||||
anomalies: List[Dict] = []
|
||||
|
||||
anomalies = []
|
||||
# 1. Flag Missing Instagram Handles in Top 50 Production Artists
|
||||
top_50 = df[df["production_hiffi_rank"] <= 50]
|
||||
for _, row in top_50.iterrows():
|
||||
if pd.isna(row.get("instagram_followers")):
|
||||
anomalies.append({
|
||||
"name": row["name"],
|
||||
"type": "TOP_50_MISSING_INSTAGRAM",
|
||||
"production_rank": row["production_hiffi_rank"],
|
||||
"value": None,
|
||||
"handle": row.get("instagram_handle"),
|
||||
"severity": "MEDIUM"
|
||||
})
|
||||
|
||||
# 1. Suspicious Low Follower Anomaly (< 10,000 followers for Top 150 production artists)
|
||||
suspicious_low = df[
|
||||
(df["production_hiffi_rank"] <= 150) &
|
||||
(df["instagram_followers"].notna()) &
|
||||
(df["instagram_followers"] < 10000)
|
||||
]
|
||||
for _, row in suspicious_low.iterrows():
|
||||
artist_name = row["name"]
|
||||
is_overridden = artist_name in self.overrides
|
||||
anomalies.append({
|
||||
"name": artist_name,
|
||||
"type": "SUSPICIOUS_LOW_FOLLOWER_COUNT",
|
||||
"production_rank": row["production_hiffi_rank"],
|
||||
"value": row["instagram_followers"],
|
||||
"handle": row.get("instagram_handle", "N/A"),
|
||||
"has_override": is_overridden,
|
||||
"severity": "HIGH" if not is_overridden else "RESOLVED_BY_OVERRIDE"
|
||||
})
|
||||
# 2. Flag Suspiciously Low Follower Counts (< 10,000) for Charted Artists
|
||||
charted = df[df["production_hiffi_rank"] <= 150]
|
||||
for _, row in charted.iterrows():
|
||||
followers = row.get("instagram_followers")
|
||||
if pd.notna(followers) and followers < 10000:
|
||||
anomalies.append({
|
||||
"name": row["name"],
|
||||
"type": "SUSPICIOUS_LOW_FOLLOWER_COUNT",
|
||||
"production_rank": row["production_hiffi_rank"],
|
||||
"value": followers,
|
||||
"handle": row.get("instagram_handle"),
|
||||
"severity": "HIGH"
|
||||
})
|
||||
|
||||
# 2. Zero-Follower Ingestion Bug (Instagram followers == 0)
|
||||
zero_followers = df[
|
||||
(df["instagram_followers"].notna()) &
|
||||
(df["instagram_followers"] == 0)
|
||||
]
|
||||
for _, row in zero_followers.iterrows():
|
||||
artist_name = row["name"]
|
||||
is_overridden = artist_name in self.overrides
|
||||
anomalies.append({
|
||||
"name": artist_name,
|
||||
"type": "ZERO_FOLLOWER_EXTRACTION_FAILURE",
|
||||
"production_rank": row.get("production_hiffi_rank", np.nan),
|
||||
"value": 0,
|
||||
"handle": row.get("instagram_handle", "N/A"),
|
||||
"has_override": is_overridden,
|
||||
"severity": "CRITICAL" if not is_overridden else "RESOLVED_BY_OVERRIDE"
|
||||
})
|
||||
# 3. Check for Data Governance Breaches (Synthetic Zeroes in Place of NaNs)
|
||||
gov_breaches = 0
|
||||
if (df["spotify_monthly_listeners"] == 0).sum() > 0:
|
||||
gov_breaches += 1
|
||||
if (df["instagram_followers"] == 0).sum() > 0:
|
||||
gov_breaches += 1
|
||||
|
||||
# 3. Missing Instagram Telemetry in Core Top 50
|
||||
top_50_missing_ig = df[
|
||||
(df["production_hiffi_rank"] <= 50) &
|
||||
(df["instagram_followers"].isna())
|
||||
]
|
||||
for _, row in top_50_missing_ig.iterrows():
|
||||
artist_name = row["name"]
|
||||
anomalies.append({
|
||||
"name": artist_name,
|
||||
"type": "TOP_50_MISSING_INSTAGRAM",
|
||||
"production_rank": row["production_hiffi_rank"],
|
||||
"value": "NaN",
|
||||
"handle": row.get("instagram_handle", "N/A"),
|
||||
"has_override": artist_name in self.overrides,
|
||||
"severity": "MEDIUM"
|
||||
})
|
||||
|
||||
# 4. Strict Streaming Governance Violations
|
||||
# A. Unattributed populated Spotify listeners (source must be kworb_public_aggregator)
|
||||
unattributed_spotify = df[
|
||||
(df["spotify_monthly_listeners"].notna()) &
|
||||
(df["spotify_data_source"] != "kworb_public_aggregator")
|
||||
]
|
||||
for _, row in unattributed_spotify.iterrows():
|
||||
anomalies.append({
|
||||
"name": row["name"],
|
||||
"type": "UNATTRIBUTED_STREAMING_DATA",
|
||||
"production_rank": row.get("production_hiffi_rank", np.nan),
|
||||
"value": row["spotify_monthly_listeners"],
|
||||
"handle": "N/A",
|
||||
"has_override": False,
|
||||
"severity": "FATAL_GOVERNANCE_BREACH"
|
||||
})
|
||||
|
||||
# B. Zero-imputed streaming metrics (Listeners == 0)
|
||||
zero_listeners = df[
|
||||
(df["spotify_monthly_listeners"].notna()) &
|
||||
(df["spotify_monthly_listeners"] <= 0)
|
||||
]
|
||||
for _, row in zero_listeners.iterrows():
|
||||
anomalies.append({
|
||||
"name": row["name"],
|
||||
"type": "ZERO_IMPUTED_STREAMING_METRIC",
|
||||
"production_rank": row.get("production_hiffi_rank", np.nan),
|
||||
"value": row["spotify_monthly_listeners"],
|
||||
"handle": "N/A",
|
||||
"has_override": False,
|
||||
"severity": "FATAL_GOVERNANCE_BREACH"
|
||||
})
|
||||
|
||||
df_anomalies = pd.DataFrame(anomalies)
|
||||
|
||||
summary = {
|
||||
"total_records_evaluated": len(df),
|
||||
"total_anomalies_flagged": len(anomalies),
|
||||
"actionable_unresolved_count": len(
|
||||
df_anomalies[df_anomalies["severity"].isin(["CRITICAL", "HIGH", "FATAL_GOVERNANCE_BREACH"])]
|
||||
) if not df_anomalies.empty else 0,
|
||||
"governance_violations": len(unattributed_spotify) + len(zero_listeners),
|
||||
"anomalies": anomalies
|
||||
anomaly_df = pd.DataFrame(anomalies)
|
||||
stats = {
|
||||
"total_rows": len(df),
|
||||
"total_anomalies": len(anomaly_df),
|
||||
"actionable_p0_p1": 0,
|
||||
"governance_breaches": gov_breaches
|
||||
}
|
||||
return summary
|
||||
|
||||
def print_diagnostic_report(self):
|
||||
"""Prints diagnostic report to console."""
|
||||
print("=" * 85)
|
||||
print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
|
||||
print("=" * 85)
|
||||
|
||||
summary = self.audit_dataset()
|
||||
print(f"Total Cohort Rows Evaluated: {summary['total_records_evaluated']}")
|
||||
print(f"Total Anomalies Flagged: {summary['total_anomalies_flagged']}")
|
||||
print(f"Actionable Unresolved (P0/P1): {summary['actionable_unresolved_count']}")
|
||||
print(f"Data Governance Breaches: {summary['governance_violations']}")
|
||||
print("-" * 85)
|
||||
|
||||
if summary["total_anomalies_flagged"] > 0:
|
||||
df_anomalies = pd.DataFrame(summary["anomalies"])
|
||||
cols = ["name", "type", "production_rank", "value", "handle", "severity"]
|
||||
print(df_anomalies[cols].to_string(index=False))
|
||||
else:
|
||||
print("Status: HEALTHY. Zero anomalies or governance breaches detected.")
|
||||
|
||||
print("=" * 85)
|
||||
return summary
|
||||
return anomaly_df, stats
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
detector = PipelineHealthDetector()
|
||||
detector.print_diagnostic_report()
|
||||
anomalies_df, summary = detector.run_health_audit()
|
||||
|
||||
print("=" * 85)
|
||||
print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
|
||||
print("=" * 85)
|
||||
print(f"Total Cohort Rows Evaluated: {summary['total_rows']}")
|
||||
print(f"Total Anomalies Flagged: {summary['total_anomalies']}")
|
||||
print(f"Actionable Unresolved (P0/P1): {summary['actionable_p0_p1']}")
|
||||
print(f"Data Governance Breaches: {summary['governance_breaches']}")
|
||||
print("-" * 85)
|
||||
if not anomalies_df.empty:
|
||||
print(anomalies_df.to_string(index=False))
|
||||
else:
|
||||
print("No anomalous signals detected across active cohort.")
|
||||
print("=" * 85)
|
||||
Reference in new issue
Block a user