hiffi-artist-intelligence/collectors/fix_youtube_ghost_channels.py

"""
collectors/fix_youtube_ghost_channels.py
Final sweep for the remaining 54 under-indexed channels.
Uses explicit verified handles/channel IDs for priority acts,
and targeted handle patterns for niche/producer acts.
"""

import os
import re
import time
import sqlite3
import logging
from pathlib import Path
from datetime import datetime, timezone
import requests
from dotenv import load_dotenv

PROJECT_ROOT = Path(__file__).resolve().parent.parent
load_dotenv(PROJECT_ROOT / ".env")

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger("YouTubeFinalSweep")

DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"
YT_API_KEY = os.getenv("YOUTUBE_API_KEY", "").strip()

# Verified official handles / IDs for acts where simple name searches fail
EXPLICIT_HANDLES = {
    "A$AP Rocky": "LIVELOVEASAPVEVO",
    "G-Eazy": "GEazyMusic",
    "GloRilla": "GloRillaOfficial",
    "Pop Smoke": "POPSMOKE",
    "Action Bronson": "ActionBronson",
    "D-Block Europe": "dblockeurope",
    "Bia": "BIAMUSIC",
    "JT": "thegirljt",
    "Takeoff": "TakeoffVEVO",
    "Too $hort": "tooshortVEVO",
    "The Alchemist": "alchemistbeats",
    "Tay Keith": "TayKeithOfficial",
    "Southside": "808mafiaworks",
    "Babyface Ray": "BabyfaceRayOfficial",
    "Hunxho": "HunxhoOfficial",
    "Unknown T": "UnknownT",
    "Tommy Cash": "TOMMYCASHWORLDWIDE",
    "The Underachievers": "TheUnderachievers",
    "Boi-1da": "Boi1da",
    "Paul Wall": "PaulWall",
    "GZA": "GZAVEVO",
    "Ghetts": "GhettsOfficial",
    "Black Thought": "BlackThoughtOfficial",
    "Big Daddy Kane": "BigDaddyKaneOfficial",
    "Cakes da Killa": "CakesDaKilla",
    "Che Noir": "CheNoir",
    "Snak The Ripper": "StealthBombRecords",
    "Z-Ro": "ZRoOfficialChannel",
    "Wheezy": "WheezyOfficial",
    "Dreezy": "DreezyVEVO",
    "Kankan": "kankan5720",
    "Osamason": "osamason",
    "ScarLip": "ScarLip",
    "Slick Rick": "SlickRickVEVO",
    "Buddy": "BuddyMusic",
    "Young Adz": "youngadz1",
    "Yung Manny": "YungManny",
    "Mike Jones": "MikeJones",
    "Quelle Chris": "QuelleChris",
    "Armani Caesar": "armanicaesar",
    "Despot": "defjux",
    "Iayze": "iayzeofficial",
    "Lil Dude": "LilDude",
    "Bizzy Bone": "BizzyBoneOfficial",
    "Moor Mother": "moormother",
    "RMC Mike": "RMCMIKE",
    "StanWill": "StanWill",
    "TrDee": "TrDee",
    "U-God": "UGodOfficial",
    "G Perico": "gpericopc"
}


def get_stats_by_channel_id(cid: str):
    url = f"https://www.googleapis.com/youtube/v3/channels?part=snippet,statistics&id={cid}&key={YT_API_KEY}"
    try:
        r = requests.get(url, timeout=8)
        if r.status_code == 200:
            items = r.json().get("items", [])
            if items:
                title = items[0]["snippet"]["title"]
                subs = int(items[0]["statistics"].get("subscriberCount", 0))
                views = int(items[0]["statistics"].get("viewCount", 0))
                return cid, title, subs, views
    except Exception as e:
        logger.error(f"Error fetching stats for {cid}: {e}")
    return None


def get_stats_by_handle(handle: str):
    clean = handle.lstrip("@").replace(" ", "")
    url = f"https://www.googleapis.com/youtube/v3/channels?part=snippet,statistics&forHandle={clean}&key={YT_API_KEY}"
    try:
        r = requests.get(url, timeout=8)
        if r.status_code == 200:
            items = r.json().get("items", [])
            if items:
                cid = items[0]["id"]
                title = items[0]["snippet"]["title"]
                subs = int(items[0]["statistics"].get("subscriberCount", 0))
                views = int(items[0]["statistics"].get("viewCount", 0))
                return cid, title, subs, views
    except Exception:
        pass
    return None


def run():
    if not YT_API_KEY:
        logger.error("YOUTUBE_API_KEY missing from .env")
        return

    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    cur.execute("""
        SELECT a.artist_id, a.artist_name, a.youtube_url, t.subscribers_count
        FROM artists a
        JOIN telemetry_metrics t ON a.artist_id = t.artist_id
        WHERE a.image_verified_flag = 1 AND t.subscribers_count < 10000
        ORDER BY a.artist_name ASC
    """)
    rows = cur.fetchall()
    logger.info(f"Loaded {len(rows)} remaining artists to remediate...")

    fixed = 0
    now_iso = datetime.now(timezone.utc).isoformat()

    for idx, (artist_id, name, old_url, old_subs) in enumerate(rows, 1):
        res = None

        # 1. Check explicit mapping dictionary
        if name in EXPLICIT_HANDLES:
            target = EXPLICIT_HANDLES[name]
            if target.startswith("UC"):
                res = get_stats_by_channel_id(target)
            else:
                res = get_stats_by_handle(target)

        # 2. Heuristic handle fallback if explicit check didn't resolve or returned low subs
        if not res or res[2] <= old_subs:
            slug = re.sub(r"[^a-zA-Z0-9]", "", name)
            fallbacks = [
                f"{slug}",
                f"{slug}TV",
                f"{slug}Music",
                f"{slug}VEVO",
                f"Official{slug}"
            ]
            for fb in fallbacks:
                res = get_stats_by_handle(fb)
                if res and res[2] > old_subs:
                    break
                time.sleep(0.04)

        # 3. Apply update if channel with more subscribers was resolved
        if res and res[2] > old_subs:
            cid, title, subs, views = res
            new_url = f"https://www.youtube.com/channel/{cid}"

            cur.execute("UPDATE artists SET youtube_url = ? WHERE artist_id = ?", (new_url, artist_id))
            cur.execute("""
                UPDATE telemetry_metrics 
                SET subscribers_count = ?, total_views = ?, last_synced_at = ? 
                WHERE artist_id = ?
            """, (subs, views, now_iso, artist_id))

            fixed += 1
            logger.info(f"✓ [{idx}/{len(rows)}] Fixed {name}: {old_subs} -> {subs:,} subs ({title})")
        else:
            logger.info(f"- [{idx}/{len(rows)}] Unchanged: {name} (kept {old_subs:,} subs)")

        if idx % 10 == 0:
            conn.commit()
        time.sleep(0.08)

    conn.commit()
    conn.close()
    logger.info(f"Sweep complete. Updated {fixed} / {len(rows)} channels.")


if __name__ == "__main__":
    run()