""" pipeline/patch_anomalies.py Patches specific Instagram handle anomalies in multiplatform_checkpoints.csv and updates checkpoints without re-running 500 artists. """ import sys import json from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) import numpy as np import pandas as pd from collectors.browser_collector import MultiPlatformSignalCollector from playwright.sync_api import sync_playwright CHECKPOINT_FILE = PROJECT_ROOT / "data" / "checkpoints" / "multiplatform_checkpoints.csv" OVERRIDES_FILE = PROJECT_ROOT / "data" / "handle_overrides.json" if not OVERRIDES_FILE.exists(): print("Overrides file not found.") sys.exit(1) with open(OVERRIDES_FILE, "r", encoding="utf-8") as f: overrides = json.load(f) df_ckpt = pd.read_csv(CHECKPOINT_FILE) # Sanitize any existing 0 follower extraction bugs to NaN df_ckpt.loc[df_ckpt["instagram_followers"] == 0, "instagram_followers"] = np.nan collector = MultiPlatformSignalCollector(headless=True) print("=" * 85) print(f"PATCHING {len(overrides)} IDENTIFIED INSTAGRAM ANOMALIES") print("=" * 85) with sync_playwright() as p: browser = p.chromium.launch(headless=True) page = browser.new_page( viewport={"width": 1920, "height": 1080}, user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ) for artist, official_handle in overrides.items(): mask = df_ckpt["name"].str.strip().str.lower() == artist.lower() if mask.any(): ig_url = f"https://www.instagram.com/{official_handle}/" res = collector.collect_artist({"name": artist, "ig_url": ig_url}, page) followers = res.get("instagram_followers") # Prevent 0-follower parse artifacts from entering checkpoint if followers is not None and followers <= 0: followers = None df_ckpt.loc[mask, "instagram_handle"] = official_handle df_ckpt.loc[mask, "instagram_followers"] = followers df_ckpt.loc[mask, "ig_status"] = "SUCCESS" if followers is not None else "AGE_GATED_OR_PRIVATE" f_str = f"{followers:,}" if followers is not None else "Unlisted/Age-Gated (NaN)" print(f"Updated: {artist} -> @{official_handle} ({f_str})") else: print(f"Artist {artist} not found in checkpoints.") browser.close() df_ckpt.to_csv(CHECKPOINT_FILE, index=False) print("\nCheckpoints successfully updated!") print("=" * 85)