65 lines
2.3 KiB
Python
65 lines
2.3 KiB
Python
|
"""
|
||
|
|
pipeline/patch_anomalies.py
|
||
|
|
Patches specific Instagram handle anomalies in multiplatform_checkpoints.csv
|
||
|
|
and regenerates the final dataset in seconds without re-running 500 artists.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import sys
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
|
||
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
|
|
if str(PROJECT_ROOT) not in sys.path:
|
||
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||
|
|
|
||
|
|
import pandas as pd
|
||
|
|
from collectors.browser_collector import MultiPlatformSignalCollector
|
||
|
|
from playwright.sync_api import sync_playwright
|
||
|
|
|
||
|
|
CHECKPOINT_FILE = PROJECT_ROOT / "data" / "checkpoints" / "multiplatform_checkpoints.csv"
|
||
|
|
OVERRIDES_FILE = PROJECT_ROOT / "data" / "handle_overrides.json"
|
||
|
|
|
||
|
|
if not OVERRIDES_FILE.exists():
|
||
|
|
print("Overrides file not found.")
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
with open(OVERRIDES_FILE, "r") as f:
|
||
|
|
overrides = json.load(f)
|
||
|
|
|
||
|
|
df_ckpt = pd.read_csv(CHECKPOINT_FILE)
|
||
|
|
collector = MultiPlatformSignalCollector(headless=True)
|
||
|
|
|
||
|
|
print("=" * 85)
|
||
|
|
print(f"PATCHING {len(overrides)} IDENTIFIED INSTAGRAM ANOMALIES")
|
||
|
|
print("=" * 85)
|
||
|
|
|
||
|
|
with sync_playwright() as p:
|
||
|
|
browser = p.chromium.launch(headless=True)
|
||
|
|
page = browser.new_page(
|
||
|
|
viewport={"width": 1920, "height": 1080},
|
||
|
|
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||
|
|
)
|
||
|
|
|
||
|
|
for artist, official_handle in overrides.items():
|
||
|
|
mask = df_ckpt["name"].str.strip().str.lower() == artist.lower()
|
||
|
|
if mask.any():
|
||
|
|
ig_url = f"https://www.instagram.com/{official_handle}/"
|
||
|
|
res = collector.collect_artist({"name": artist, "ig_url": ig_url}, page)
|
||
|
|
followers = res.get("instagram_followers")
|
||
|
|
|
||
|
|
df_ckpt.loc[mask, "instagram_handle"] = official_handle
|
||
|
|
df_ckpt.loc[mask, "instagram_followers"] = followers
|
||
|
|
df_ckpt.loc[mask, "ig_status"] = res.get("ig_status", "SUCCESS")
|
||
|
|
|
||
|
|
f_str = f"{followers:,}" if followers is not None else "Failed"
|
||
|
|
print(f"Updated: {artist} -> @{official_handle} ({f_str} followers)")
|
||
|
|
else:
|
||
|
|
print(f"Artist {artist} not found in checkpoints.")
|
||
|
|
|
||
|
|
browser.close()
|
||
|
|
|
||
|
|
# Save updated checkpoints
|
||
|
|
df_ckpt.to_csv(CHECKPOINT_FILE, index=False)
|
||
|
|
print("\nCheckpoints successfully updated!")
|
||
|
|
print("=" * 85)
|