""" collectors/fix_youtube_ghost_channels.py Final sweep for the remaining 54 under-indexed channels. Uses explicit verified handles/channel IDs for priority acts, and targeted handle patterns for niche/producer acts. """ import os import re import time import sqlite3 import logging from pathlib import Path from datetime import datetime, timezone import requests from dotenv import load_dotenv PROJECT_ROOT = Path(__file__).resolve().parent.parent load_dotenv(PROJECT_ROOT / ".env") logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("YouTubeFinalSweep") DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db" YT_API_KEY = os.getenv("YOUTUBE_API_KEY", "").strip() # Verified official handles / IDs for acts where simple name searches fail EXPLICIT_HANDLES = { "A$AP Rocky": "LIVELOVEASAPVEVO", "G-Eazy": "GEazyMusic", "GloRilla": "GloRillaOfficial", "Pop Smoke": "POPSMOKE", "Action Bronson": "ActionBronson", "D-Block Europe": "dblockeurope", "Bia": "BIAMUSIC", "JT": "thegirljt", "Takeoff": "TakeoffVEVO", "Too $hort": "tooshortVEVO", "The Alchemist": "alchemistbeats", "Tay Keith": "TayKeithOfficial", "Southside": "808mafiaworks", "Babyface Ray": "BabyfaceRayOfficial", "Hunxho": "HunxhoOfficial", "Unknown T": "UnknownT", "Tommy Cash": "TOMMYCASHWORLDWIDE", "The Underachievers": "TheUnderachievers", "Boi-1da": "Boi1da", "Paul Wall": "PaulWall", "GZA": "GZAVEVO", "Ghetts": "GhettsOfficial", "Black Thought": "BlackThoughtOfficial", "Big Daddy Kane": "BigDaddyKaneOfficial", "Cakes da Killa": "CakesDaKilla", "Che Noir": "CheNoir", "Snak The Ripper": "StealthBombRecords", "Z-Ro": "ZRoOfficialChannel", "Wheezy": "WheezyOfficial", "Dreezy": "DreezyVEVO", "Kankan": "kankan5720", "Osamason": "osamason", "ScarLip": "ScarLip", "Slick Rick": "SlickRickVEVO", "Buddy": "BuddyMusic", "Young Adz": "youngadz1", "Yung Manny": "YungManny", "Mike Jones": "MikeJones", "Quelle Chris": "QuelleChris", "Armani Caesar": "armanicaesar", "Despot": "defjux", "Iayze": "iayzeofficial", "Lil Dude": "LilDude", "Bizzy Bone": "BizzyBoneOfficial", "Moor Mother": "moormother", "RMC Mike": "RMCMIKE", "StanWill": "StanWill", "TrDee": "TrDee", "U-God": "UGodOfficial", "G Perico": "gpericopc" } def get_stats_by_channel_id(cid: str): url = f"https://www.googleapis.com/youtube/v3/channels?part=snippet,statistics&id={cid}&key={YT_API_KEY}" try: r = requests.get(url, timeout=8) if r.status_code == 200: items = r.json().get("items", []) if items: title = items[0]["snippet"]["title"] subs = int(items[0]["statistics"].get("subscriberCount", 0)) views = int(items[0]["statistics"].get("viewCount", 0)) return cid, title, subs, views except Exception as e: logger.error(f"Error fetching stats for {cid}: {e}") return None def get_stats_by_handle(handle: str): clean = handle.lstrip("@").replace(" ", "") url = f"https://www.googleapis.com/youtube/v3/channels?part=snippet,statistics&forHandle={clean}&key={YT_API_KEY}" try: r = requests.get(url, timeout=8) if r.status_code == 200: items = r.json().get("items", []) if items: cid = items[0]["id"] title = items[0]["snippet"]["title"] subs = int(items[0]["statistics"].get("subscriberCount", 0)) views = int(items[0]["statistics"].get("viewCount", 0)) return cid, title, subs, views except Exception: pass return None def run(): if not YT_API_KEY: logger.error("YOUTUBE_API_KEY missing from .env") return conn = sqlite3.connect(DB_PATH) cur = conn.cursor() cur.execute(""" SELECT a.artist_id, a.artist_name, a.youtube_url, t.subscribers_count FROM artists a JOIN telemetry_metrics t ON a.artist_id = t.artist_id WHERE a.image_verified_flag = 1 AND t.subscribers_count < 10000 ORDER BY a.artist_name ASC """) rows = cur.fetchall() logger.info(f"Loaded {len(rows)} remaining artists to remediate...") fixed = 0 now_iso = datetime.now(timezone.utc).isoformat() for idx, (artist_id, name, old_url, old_subs) in enumerate(rows, 1): res = None # 1. Check explicit mapping dictionary if name in EXPLICIT_HANDLES: target = EXPLICIT_HANDLES[name] if target.startswith("UC"): res = get_stats_by_channel_id(target) else: res = get_stats_by_handle(target) # 2. Heuristic handle fallback if explicit check didn't resolve or returned low subs if not res or res[2] <= old_subs: slug = re.sub(r"[^a-zA-Z0-9]", "", name) fallbacks = [ f"{slug}", f"{slug}TV", f"{slug}Music", f"{slug}VEVO", f"Official{slug}" ] for fb in fallbacks: res = get_stats_by_handle(fb) if res and res[2] > old_subs: break time.sleep(0.04) # 3. Apply update if channel with more subscribers was resolved if res and res[2] > old_subs: cid, title, subs, views = res new_url = f"https://www.youtube.com/channel/{cid}" cur.execute("UPDATE artists SET youtube_url = ? WHERE artist_id = ?", (new_url, artist_id)) cur.execute(""" UPDATE telemetry_metrics SET subscribers_count = ?, total_views = ?, last_synced_at = ? WHERE artist_id = ? """, (subs, views, now_iso, artist_id)) fixed += 1 logger.info(f"✓ [{idx}/{len(rows)}] Fixed {name}: {old_subs} -> {subs:,} subs ({title})") else: logger.info(f"- [{idx}/{len(rows)}] Unchanged: {name} (kept {old_subs:,} subs)") if idx % 10 == 0: conn.commit() time.sleep(0.08) conn.commit() conn.close() logger.info(f"Sweep complete. Updated {fixed} / {len(rows)} channels.") if __name__ == "__main__": run()