feat: complete multiplatform pipeline hardening, neutral anchor scoring, and automated health checks
This commit is contained in:
Srilekha committed
1 parent
56cbd08248
commit
66db687a58
11 files changed
+1097
-1005
No files matched your search
+164
-69
@@ -1,87 +1,182 @@
|
||||
"""
|
||||
core/health_detector.py
|
||||
Identifies inactive, legacy, zombie, and hyper-viral channels
|
||||
based on engagement-to-audience ratios and upload cadence.
|
||||
Automated Anomaly Detection & Data Health Diagnostics for HIFFI Pipeline.
|
||||
|
||||
Evaluates:
|
||||
- High-velocity artists mapped to low-follower or inactive social profiles.
|
||||
- Unextracted zero-follower anomalies (e.g. rate-limit or UI layout shifts).
|
||||
- Missing telemetry on Top 50 cohort.
|
||||
- Strict streaming governance (valid source attribution, zero-imputation bans).
|
||||
- Auto-matching against data/handle_overrides.json.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any
|
||||
import numpy as np
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
class ChannelHealthDetector:
|
||||
"""
|
||||
Evaluates channel vitality by examining daily velocity against subscriber scale.
|
||||
"""
|
||||
class PipelineHealthDetector:
|
||||
def __init__(
|
||||
self,
|
||||
dataset_path: Path = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv",
|
||||
overrides_path: Path = PROJECT_ROOT / "data" / "handle_overrides.json"
|
||||
):
|
||||
self.dataset_path = Path(dataset_path)
|
||||
self.overrides_path = Path(overrides_path)
|
||||
self.overrides = self._load_overrides()
|
||||
|
||||
def __init__(self, zombie_sub_threshold: int = 25000, zombie_velocity_cutoff: float = 100.0):
|
||||
self.zombie_sub_threshold = zombie_sub_threshold
|
||||
self.zombie_velocity_cutoff = zombie_velocity_cutoff
|
||||
def _load_overrides(self) -> Dict[str, str]:
|
||||
if self.overrides_path.exists():
|
||||
try:
|
||||
with open(self.overrides_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
return {}
|
||||
|
||||
def classify_channel(self, row: pd.Series) -> Dict[str, Any]:
|
||||
subs = row.get("subscriber_count")
|
||||
daily_vel = row.get("daily_view_velocity")
|
||||
status_flag = row.get("status_flag", "")
|
||||
|
||||
# Handle unresolvable or missing data
|
||||
if pd.isna(subs) or status_flag == "FAILED_LOOKUP":
|
||||
return {
|
||||
"health_status": "UNRESOLVED",
|
||||
"is_zombie": False,
|
||||
"health_notes": "Channel could not be resolved via YouTube API."
|
||||
}
|
||||
def audit_dataset(self, df: pd.DataFrame = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Executes an automated multi-point anomaly & governance check.
|
||||
"""
|
||||
if df is None:
|
||||
if not self.dataset_path.exists():
|
||||
raise FileNotFoundError(f"Dataset not found at {self.dataset_path}")
|
||||
df = pd.read_csv(self.dataset_path)
|
||||
|
||||
subs = float(subs)
|
||||
daily_vel = float(daily_vel) if pd.notna(daily_vel) else 0.0
|
||||
anomalies = []
|
||||
|
||||
engagement_ratio = daily_vel / max(subs, 1000.0)
|
||||
# 1. Suspicious Low Follower Anomaly (< 10,000 followers for Top 150 production artists)
|
||||
suspicious_low = df[
|
||||
(df["production_hiffi_rank"] <= 150) &
|
||||
(df["instagram_followers"].notna()) &
|
||||
(df["instagram_followers"] < 10000)
|
||||
]
|
||||
for _, row in suspicious_low.iterrows():
|
||||
artist_name = row["name"]
|
||||
is_overridden = artist_name in self.overrides
|
||||
anomalies.append({
|
||||
"name": artist_name,
|
||||
"type": "SUSPICIOUS_LOW_FOLLOWER_COUNT",
|
||||
"production_rank": row["production_hiffi_rank"],
|
||||
"value": row["instagram_followers"],
|
||||
"handle": row.get("instagram_handle", "N/A"),
|
||||
"has_override": is_overridden,
|
||||
"severity": "HIGH" if not is_overridden else "RESOLVED_BY_OVERRIDE"
|
||||
})
|
||||
|
||||
# 1. Zombie Channel: Large subscriber base with dead viewership
|
||||
if subs >= self.zombie_sub_threshold and daily_vel < self.zombie_velocity_cutoff:
|
||||
return {
|
||||
"health_status": "ZOMBIE_CHANNEL",
|
||||
"is_zombie": True,
|
||||
"health_notes": f"High subscriber base ({int(subs):,}) with negligible viewership ({daily_vel:.1f} views/day)."
|
||||
}
|
||||
# 2. Zero-Follower Ingestion Bug (Instagram followers == 0)
|
||||
zero_followers = df[
|
||||
(df["instagram_followers"].notna()) &
|
||||
(df["instagram_followers"] == 0)
|
||||
]
|
||||
for _, row in zero_followers.iterrows():
|
||||
artist_name = row["name"]
|
||||
is_overridden = artist_name in self.overrides
|
||||
anomalies.append({
|
||||
"name": artist_name,
|
||||
"type": "ZERO_FOLLOWER_EXTRACTION_FAILURE",
|
||||
"production_rank": row.get("production_hiffi_rank", np.nan),
|
||||
"value": 0,
|
||||
"handle": row.get("instagram_handle", "N/A"),
|
||||
"has_override": is_overridden,
|
||||
"severity": "CRITICAL" if not is_overridden else "RESOLVED_BY_OVERRIDE"
|
||||
})
|
||||
|
||||
# 2. Dormant Catalog: No recent uploads and negligible velocity
|
||||
if status_flag == "INACTIVE_OR_DORMANT" or daily_vel == 0.0:
|
||||
return {
|
||||
"health_status": "DORMANT_CATALOG",
|
||||
"is_zombie": subs >= 10000,
|
||||
"health_notes": "Zero recent uploads and stalled catalog velocity."
|
||||
}
|
||||
# 3. Missing Instagram Telemetry in Core Top 50
|
||||
top_50_missing_ig = df[
|
||||
(df["production_hiffi_rank"] <= 50) &
|
||||
(df["instagram_followers"].isna())
|
||||
]
|
||||
for _, row in top_50_missing_ig.iterrows():
|
||||
artist_name = row["name"]
|
||||
anomalies.append({
|
||||
"name": artist_name,
|
||||
"type": "TOP_50_MISSING_INSTAGRAM",
|
||||
"production_rank": row["production_hiffi_rank"],
|
||||
"value": "NaN",
|
||||
"handle": row.get("instagram_handle", "N/A"),
|
||||
"has_override": artist_name in self.overrides,
|
||||
"severity": "MEDIUM"
|
||||
})
|
||||
|
||||
# 3. Viral Surge: Recent views exceed subscriber base
|
||||
if engagement_ratio >= 1.0:
|
||||
return {
|
||||
"health_status": "VIRAL_SURGE",
|
||||
"is_zombie": False,
|
||||
"health_notes": "Audience reach is exploding beyond existing subscriber base."
|
||||
}
|
||||
# 4. Strict Streaming Governance Violations
|
||||
# A. Unattributed populated Spotify listeners (source must be kworb_public_aggregator)
|
||||
unattributed_spotify = df[
|
||||
(df["spotify_monthly_listeners"].notna()) &
|
||||
(df["spotify_data_source"] != "kworb_public_aggregator")
|
||||
]
|
||||
for _, row in unattributed_spotify.iterrows():
|
||||
anomalies.append({
|
||||
"name": row["name"],
|
||||
"type": "UNATTRIBUTED_STREAMING_DATA",
|
||||
"production_rank": row.get("production_hiffi_rank", np.nan),
|
||||
"value": row["spotify_monthly_listeners"],
|
||||
"handle": "N/A",
|
||||
"has_override": False,
|
||||
"severity": "FATAL_GOVERNANCE_BREACH"
|
||||
})
|
||||
|
||||
# 4. Catalog Evergreen: Big catalog keeping views high despite no new release
|
||||
if status_flag == "CATALOG_ACTIVE_NO_RECENT_UPLOAD":
|
||||
return {
|
||||
"health_status": "CATALOG_EVERGREEN",
|
||||
"is_zombie": False,
|
||||
"health_notes": "Strong passive catalog views despite release pause."
|
||||
}
|
||||
# B. Zero-imputed streaming metrics (Listeners == 0)
|
||||
zero_listeners = df[
|
||||
(df["spotify_monthly_listeners"].notna()) &
|
||||
(df["spotify_monthly_listeners"] <= 0)
|
||||
]
|
||||
for _, row in zero_listeners.iterrows():
|
||||
anomalies.append({
|
||||
"name": row["name"],
|
||||
"type": "ZERO_IMPUTED_STREAMING_METRIC",
|
||||
"production_rank": row.get("production_hiffi_rank", np.nan),
|
||||
"value": row["spotify_monthly_listeners"],
|
||||
"handle": "N/A",
|
||||
"has_override": False,
|
||||
"severity": "FATAL_GOVERNANCE_BREACH"
|
||||
})
|
||||
|
||||
# 5. Healthy Active Channel
|
||||
return {
|
||||
"health_status": "ACTIVE_HEALTHY",
|
||||
"is_zombie": False,
|
||||
"health_notes": "Consistent release schedule and steady audience engagement."
|
||||
df_anomalies = pd.DataFrame(anomalies)
|
||||
|
||||
summary = {
|
||||
"total_records_evaluated": len(df),
|
||||
"total_anomalies_flagged": len(anomalies),
|
||||
"actionable_unresolved_count": len(
|
||||
df_anomalies[df_anomalies["severity"].isin(["CRITICAL", "HIGH", "FATAL_GOVERNANCE_BREACH"])]
|
||||
) if not df_anomalies.empty else 0,
|
||||
"governance_violations": len(unattributed_spotify) + len(zero_listeners),
|
||||
"anomalies": anomalies
|
||||
}
|
||||
return summary
|
||||
|
||||
def evaluate_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Applies health classification across all artist records."""
|
||||
out = df.copy()
|
||||
health_results = out.apply(self.classify_channel, axis=1)
|
||||
|
||||
out["health_status"] = [r["health_status"] for r in health_results]
|
||||
out["is_zombie"] = [r["is_zombie"] for r in health_results]
|
||||
out["health_notes"] = [r["health_notes"] for r in health_results]
|
||||
|
||||
return out
|
||||
def print_diagnostic_report(self):
|
||||
"""Prints diagnostic report to console."""
|
||||
print("=" * 85)
|
||||
print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
|
||||
print("=" * 85)
|
||||
|
||||
summary = self.audit_dataset()
|
||||
print(f"Total Cohort Rows Evaluated: {summary['total_records_evaluated']}")
|
||||
print(f"Total Anomalies Flagged: {summary['total_anomalies_flagged']}")
|
||||
print(f"Actionable Unresolved (P0/P1): {summary['actionable_unresolved_count']}")
|
||||
print(f"Data Governance Breaches: {summary['governance_violations']}")
|
||||
print("-" * 85)
|
||||
|
||||
if summary["total_anomalies_flagged"] > 0:
|
||||
df_anomalies = pd.DataFrame(summary["anomalies"])
|
||||
cols = ["name", "type", "production_rank", "value", "handle", "severity"]
|
||||
print(df_anomalies[cols].to_string(index=False))
|
||||
else:
|
||||
print("Status: HEALTHY. Zero anomalies or governance breaches detected.")
|
||||
|
||||
print("=" * 85)
|
||||
return summary
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
detector = PipelineHealthDetector()
|
||||
detector.print_diagnostic_report()
|
||||
Reference in new issue
Block a user