import os from pathlib import Path import sqlite3 import time import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from dotenv import load_dotenv PROJECT_ROOT = Path(__file__).resolve().parent.parent load_dotenv(PROJECT_ROOT / ".env") YOUTUBE_API_KEY = os.getenv("YOUTUBE_API_KEY", "").strip() DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db" if not YOUTUBE_API_KEY: print("Error: YOUTUBE_API_KEY not found in .env") exit(1) # Configure resilient HTTP session with automatic retries for WinError 10054 session = requests.Session() retry_strategy = Retry( total=4, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504], ) adapter = HTTPAdapter(max_retries=retry_strategy) session.mount("https://", adapter) session.mount("http://", adapter) # Verified official YouTube handles VERIFIED_HANDLES = { # Mismatches "Young Thug": "YoungThug", "Bas": "bas", "Cardi B": "iamcardib", "Stormzy": "StormzyTV", "Chingy": "Chingy", "Cassidy": "CassidyLarsiny", "Madlib": "MadlibInvazion", "EARTHGANG": "EARTHGANG", "Juvenile": "Juvenile", "UGK": "UGKMusic", "Conway the Machine": "WhoIsConway", "Smoke DZA": "SmokeDZA", "Open Mike Eagle": "mikeeagletv", # Remaining unlinked cohort "Key Glock": "KeyGlock", "Deante' Hitchcock": "DeanteHitchcock", "Veeze": "veeze__", "Fredo Santana": "FredoSantana", "Dizzee Rascal": "dizzeerascal", "Aitch": "AitchOfficial", "Digga D": "DiggaD", "K-Trap": "ktrap_official", "Nines": "Nines", "Potter Payper": "PotterPayper", "MoStack": "MoStack", "Stefflon Don": "stefflondon", "Merkules": "Merkules", "Night Lovell": "NightLovell", "Yung Lean": "yunglean", "Bladee": "bladee", "Ecco2k": "ecco2k", "Thaiboy Digital": "thaiboydigital", "BBNO$": "bbnomula", "Jhené Aiko": "JheneAiko", "Fredo": "Fredo", "Mick Jenkins": "MickJenkins", "Kirk Knight": "KirkKnight", "Cozz": "Cozz", "Masta Killa": "MastaKilla", "Mannie Fresh": "MannieFresh", "Grip": "Grip_SS", "Yung Miami": "YungMiami", "Oshun": "OSHUniverse", "Goonew": "Goonew64", "Big Pokey": "BigPokey", "Q Da Fool": "QDaFool", "100k Track": "100kTrack", "Hak": "haknyc", "Sporting Life": "sportinglife", "KrispyLife Kidd": "krispylife_kidd", "K-os": "kosinfluence", "Dom Genesis": "DomsGenesis", "BabySmoove": "babysmoove" } def resolve_by_handle(handle: str): clean_handle = handle.lstrip("@") url = "https://www.googleapis.com/youtube/v3/channels" params = { "part": "snippet,statistics", "forHandle": clean_handle, "key": YOUTUBE_API_KEY } try: resp = session.get(url, params=params, timeout=12) if resp.status_code == 200: items = resp.json().get("items", []) if items: channel_id = items[0]["id"] stats = items[0].get("statistics", {}) subs = int(stats.get("subscriberCount", 0)) title = items[0]["snippet"].get("title", "") return channel_id, subs, title except Exception as e: print(f" [WARN: handle query failed for {handle}]: {e}") return None, 0, None conn = sqlite3.connect(DB_PATH) cur = conn.cursor() print(f"Resolving {len(VERIFIED_HANDLES)} artists via official YouTube handles...\n") success = 0 for artist_name, handle in VERIFIED_HANDLES.items(): channel_id, subs, title = resolve_by_handle(handle) if not channel_id: # Fallback to search if handle query is empty try: s_resp = session.get( "https://www.googleapis.com/youtube/v3/search", params={"part": "snippet", "q": f"{artist_name} Official", "type": "channel", "maxResults": 1, "key": YOUTUBE_API_KEY}, timeout=12 ) if s_resp.status_code == 200 and s_resp.json().get("items"): channel_id = s_resp.json()["items"][0]["snippet"]["channelId"] title = s_resp.json()["items"][0]["snippet"]["title"] stat_resp = session.get( "https://www.googleapis.com/youtube/v3/channels", params={"part": "statistics", "id": channel_id, "key": YOUTUBE_API_KEY}, timeout=12 ) if stat_resp.status_code == 200 and stat_resp.json().get("items"): subs = int(stat_resp.json()["items"][0]["statistics"].get("subscriberCount", 0)) except Exception as e: print(f" [WARN: search query failed for {artist_name}]: {e}") if channel_id: canonical_url = f"https://www.youtube.com/channel/{channel_id}" cur.execute("SELECT artist_id FROM artists WHERE LOWER(artist_name) = ?", (artist_name.lower(),)) row = cur.fetchone() if row: a_id = row[0] cur.execute("UPDATE artists SET youtube_url = ? WHERE artist_id = ?", (canonical_url, a_id)) if subs > 0: cur.execute("UPDATE telemetry_metrics SET subscribers_count = ? WHERE artist_id = ?", (subs, a_id)) conn.commit() success += 1 print(f"[{success:02d}/{len(VERIFIED_HANDLES)}] {artist_name:<20} -> {title} (Subs: {subs:,}) [{channel_id}]") else: print(f"[NOT IN DB] {artist_name}") else: print(f"[FAILED] {artist_name}") time.sleep(0.3) conn.close() print(f"\nCompleted: {success}/{len(VERIFIED_HANDLES)} channels patched in hiffi_catalog.db.")