feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort

collectors/fix_youtube_ghost_channels.py

@@ -0,0 +1,191 @@
+"""
+collectors/fix_youtube_ghost_channels.py
+Final sweep for the remaining 54 under-indexed channels.
+Uses explicit verified handles/channel IDs for priority acts,
+and targeted handle patterns for niche/producer acts.
+"""
+
+import os
+import re
+import time
+import sqlite3
+import logging
+from pathlib import Path
+from datetime import datetime, timezone
+import requests
+from dotenv import load_dotenv
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+load_dotenv(PROJECT_ROOT / ".env")
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger("YouTubeFinalSweep")
+
+DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"
+YT_API_KEY = os.getenv("YOUTUBE_API_KEY", "").strip()
+
+# Verified official handles / IDs for acts where simple name searches fail
+EXPLICIT_HANDLES = {
+    "A$AP Rocky": "LIVELOVEASAPVEVO",
+    "G-Eazy": "GEazyMusic",
+    "GloRilla": "GloRillaOfficial",
+    "Pop Smoke": "POPSMOKE",
+    "Action Bronson": "ActionBronson",
+    "D-Block Europe": "dblockeurope",
+    "Bia": "BIAMUSIC",
+    "JT": "thegirljt",
+    "Takeoff": "TakeoffVEVO",
+    "Too $hort": "tooshortVEVO",
+    "The Alchemist": "alchemistbeats",
+    "Tay Keith": "TayKeithOfficial",
+    "Southside": "808mafiaworks",
+    "Babyface Ray": "BabyfaceRayOfficial",
+    "Hunxho": "HunxhoOfficial",
+    "Unknown T": "UnknownT",
+    "Tommy Cash": "TOMMYCASHWORLDWIDE",
+    "The Underachievers": "TheUnderachievers",
+    "Boi-1da": "Boi1da",
+    "Paul Wall": "PaulWall",
+    "GZA": "GZAVEVO",
+    "Ghetts": "GhettsOfficial",
+    "Black Thought": "BlackThoughtOfficial",
+    "Big Daddy Kane": "BigDaddyKaneOfficial",
+    "Cakes da Killa": "CakesDaKilla",
+    "Che Noir": "CheNoir",
+    "Snak The Ripper": "StealthBombRecords",
+    "Z-Ro": "ZRoOfficialChannel",
+    "Wheezy": "WheezyOfficial",
+    "Dreezy": "DreezyVEVO",
+    "Kankan": "kankan5720",
+    "Osamason": "osamason",
+    "ScarLip": "ScarLip",
+    "Slick Rick": "SlickRickVEVO",
+    "Buddy": "BuddyMusic",
+    "Young Adz": "youngadz1",
+    "Yung Manny": "YungManny",
+    "Mike Jones": "MikeJones",
+    "Quelle Chris": "QuelleChris",
+    "Armani Caesar": "armanicaesar",
+    "Despot": "defjux",
+    "Iayze": "iayzeofficial",
+    "Lil Dude": "LilDude",
+    "Bizzy Bone": "BizzyBoneOfficial",
+    "Moor Mother": "moormother",
+    "RMC Mike": "RMCMIKE",
+    "StanWill": "StanWill",
+    "TrDee": "TrDee",
+    "U-God": "UGodOfficial",
+    "G Perico": "gpericopc"
+}
+
+
+def get_stats_by_channel_id(cid: str):
+    url = f"https://www.googleapis.com/youtube/v3/channels?part=snippet,statistics&id={cid}&key={YT_API_KEY}"
+    try:
+        r = requests.get(url, timeout=8)
+        if r.status_code == 200:
+            items = r.json().get("items", [])
+            if items:
+                title = items[0]["snippet"]["title"]
+                subs = int(items[0]["statistics"].get("subscriberCount", 0))
+                views = int(items[0]["statistics"].get("viewCount", 0))
+                return cid, title, subs, views
+    except Exception as e:
+        logger.error(f"Error fetching stats for {cid}: {e}")
+    return None
+
+
+def get_stats_by_handle(handle: str):
+    clean = handle.lstrip("@").replace(" ", "")
+    url = f"https://www.googleapis.com/youtube/v3/channels?part=snippet,statistics&forHandle={clean}&key={YT_API_KEY}"
+    try:
+        r = requests.get(url, timeout=8)
+        if r.status_code == 200:
+            items = r.json().get("items", [])
+            if items:
+                cid = items[0]["id"]
+                title = items[0]["snippet"]["title"]
+                subs = int(items[0]["statistics"].get("subscriberCount", 0))
+                views = int(items[0]["statistics"].get("viewCount", 0))
+                return cid, title, subs, views
+    except Exception:
+        pass
+    return None
+
+
+def run():
+    if not YT_API_KEY:
+        logger.error("YOUTUBE_API_KEY missing from .env")
+        return
+
+    conn = sqlite3.connect(DB_PATH)
+    cur = conn.cursor()
+
+    cur.execute("""
+        SELECT a.artist_id, a.artist_name, a.youtube_url, t.subscribers_count
+        FROM artists a
+        JOIN telemetry_metrics t ON a.artist_id = t.artist_id
+        WHERE a.image_verified_flag = 1 AND t.subscribers_count < 10000
+        ORDER BY a.artist_name ASC
+    """)
+    rows = cur.fetchall()
+    logger.info(f"Loaded {len(rows)} remaining artists to remediate...")
+
+    fixed = 0
+    now_iso = datetime.now(timezone.utc).isoformat()
+
+    for idx, (artist_id, name, old_url, old_subs) in enumerate(rows, 1):
+        res = None
+
+        # 1. Check explicit mapping dictionary
+        if name in EXPLICIT_HANDLES:
+            target = EXPLICIT_HANDLES[name]
+            if target.startswith("UC"):
+                res = get_stats_by_channel_id(target)
+            else:
+                res = get_stats_by_handle(target)
+
+        # 2. Heuristic handle fallback if explicit check didn't resolve or returned low subs
+        if not res or res[2] <= old_subs:
+            slug = re.sub(r"[^a-zA-Z0-9]", "", name)
+            fallbacks = [
+                f"{slug}",
+                f"{slug}TV",
+                f"{slug}Music",
+                f"{slug}VEVO",
+                f"Official{slug}"
+            ]
+            for fb in fallbacks:
+                res = get_stats_by_handle(fb)
+                if res and res[2] > old_subs:
+                    break
+                time.sleep(0.04)
+
+        # 3. Apply update if channel with more subscribers was resolved
+        if res and res[2] > old_subs:
+            cid, title, subs, views = res
+            new_url = f"https://www.youtube.com/channel/{cid}"
+
+            cur.execute("UPDATE artists SET youtube_url = ? WHERE artist_id = ?", (new_url, artist_id))
+            cur.execute("""
+                UPDATE telemetry_metrics 
+                SET subscribers_count = ?, total_views = ?, last_synced_at = ? 
+                WHERE artist_id = ?
+            """, (subs, views, now_iso, artist_id))
+
+            fixed += 1
+            logger.info(f"✓ [{idx}/{len(rows)}] Fixed {name}: {old_subs} -> {subs:,} subs ({title})")
+        else:
+            logger.info(f"- [{idx}/{len(rows)}] Unchanged: {name} (kept {old_subs:,} subs)")
+
+        if idx % 10 == 0:
+            conn.commit()
+        time.sleep(0.08)
+
+    conn.commit()
+    conn.close()
+    logger.info(f"Sweep complete. Updated {fixed} / {len(rows)} channels.")
+
+
+if __name__ == "__main__":
+    run()