feat: complete multiplatform pipeline hardening, neutral anchor scoring, and automated health checks

pipeline/patch_anomalies.py

@@ -1,18 +1,18 @@
 """
 pipeline/patch_anomalies.py
 Patches specific Instagram handle anomalies in multiplatform_checkpoints.csv
-and regenerates the final dataset in seconds without re-running 500 artists.
+and updates checkpoints without re-running 500 artists.
 """
 
 import sys
 import json
 from pathlib import Path
-from datetime import datetime, timezone
 
 PROJECT_ROOT = Path(__file__).resolve().parent.parent
 if str(PROJECT_ROOT) not in sys.path:
     sys.path.insert(0, str(PROJECT_ROOT))
 
+import numpy as np
 import pandas as pd
 from collectors.browser_collector import MultiPlatformSignalCollector
 from playwright.sync_api import sync_playwright
@@ -24,10 +24,14 @@ if not OVERRIDES_FILE.exists():
     print("Overrides file not found.")
     sys.exit(1)
 
-with open(OVERRIDES_FILE, "r") as f:
+with open(OVERRIDES_FILE, "r", encoding="utf-8") as f:
     overrides = json.load(f)
 
 df_ckpt = pd.read_csv(CHECKPOINT_FILE)
+
+# Sanitize any existing 0 follower extraction bugs to NaN
+df_ckpt.loc[df_ckpt["instagram_followers"] == 0, "instagram_followers"] = np.nan
+
 collector = MultiPlatformSignalCollector(headless=True)
 
 print("=" * 85)
@@ -47,19 +51,22 @@ with sync_playwright() as p:
             ig_url = f"https://www.instagram.com/{official_handle}/"
             res = collector.collect_artist({"name": artist, "ig_url": ig_url}, page)
             followers = res.get("instagram_followers")
-            
+
+            # Prevent 0-follower parse artifacts from entering checkpoint
+            if followers is not None and followers <= 0:
+                followers = None
+
             df_ckpt.loc[mask, "instagram_handle"] = official_handle
             df_ckpt.loc[mask, "instagram_followers"] = followers
-            df_ckpt.loc[mask, "ig_status"] = res.get("ig_status", "SUCCESS")
-            
-            f_str = f"{followers:,}" if followers is not None else "Failed"
-            print(f"Updated: {artist} -> @{official_handle} ({f_str} followers)")
+            df_ckpt.loc[mask, "ig_status"] = "SUCCESS" if followers is not None else "AGE_GATED_OR_PRIVATE"
+
+            f_str = f"{followers:,}" if followers is not None else "Unlisted/Age-Gated (NaN)"
+            print(f"Updated: {artist} -> @{official_handle} ({f_str})")
         else:
             print(f"Artist {artist} not found in checkpoints.")
 
     browser.close()
 
-# Save updated checkpoints
 df_ckpt.to_csv(CHECKPOINT_FILE, index=False)
 print("\nCheckpoints successfully updated!")
 print("=" * 85)