feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort

scripts/scrape_live_youtube_151.py

@@ -0,0 +1,114 @@
+"""
+scripts/scrape_live_youtube_151.py
+Scrapes live, ground-truth subscriber counts, views, and videos directly
+from YouTube for the 151 unresolved artists using canonical channel mappings.
+"""
+
+import json
+import re
+import sys
+import time
+from pathlib import Path
+import requests
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from core.database import get_connection
+
+MAP_FILE = ROOT_DIR / "data" / "youtube_channel_mappings.json"
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+
+def parse_metric(raw_text):
+    if not raw_text:
+        return 0
+    t = raw_text.upper().replace(",", "").strip()
+    match = re.search(r"([\d\.]+)\s*([KMB])?", t)
+    if not match:
+        return 0
+    val = float(match.group(1))
+    suffix = match.group(2)
+    if suffix == "K":
+        val *= 1e3
+    elif suffix == "M":
+        val *= 1e6
+    elif suffix == "B":
+        val *= 1e9
+    return int(val)
+
+
+def main():
+    if not MAP_FILE.exists():
+        print(f"Mapping file not found at {MAP_FILE}")
+        return
+
+    with open(MAP_FILE, "r", encoding="utf-8") as f:
+        mappings = json.load(f)
+
+    conn = get_connection()
+    cur = conn.cursor()
+
+    print(f"Targeting {len(mappings)} channels for live ground-truth extraction...")
+    success = 0
+
+    for idx, (artist, info) in enumerate(mappings.items(), 1):
+        url = info.get("youtube", "").rstrip("/")
+        if not url:
+            continue
+
+        target_url = url if "/channel/" in url else url
+
+        try:
+            resp = requests.get(target_url, headers=HEADERS, timeout=12)
+            html = resp.text
+
+            # Match subscriber text in initial data payload
+            sub_match = re.search(r'"subscriberCountText":\{.*?"simpleText":"([^"]+)"', html)
+            if not sub_match:
+                sub_match = re.search(r'"accessibilityData":\{"label":"([^"]+subscribers?)"', html)
+
+            view_match = re.search(r'"viewCountText":\{.*?"simpleText":"([^"]+)"', html)
+            vid_match = re.search(r'"videoCountText":\{.*?"text":"([^"]+)"', html)
+
+            subs = parse_metric(sub_match.group(1)) if sub_match else 0
+            views = parse_metric(view_match.group(1)) if view_match else 0
+            vids = parse_metric(vid_match.group(1)) if vid_match else 0
+
+            if subs > 0:
+                success += 1
+                cur.execute(
+                    """
+                    UPDATE telemetry_metrics
+                    SET subscribers_count = ?
+                    WHERE artist_id IN (
+                        SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
+                    )
+                    """,
+                    (subs, artist),
+                )
+                print(f"[{idx}/{len(mappings)}] {artist}: {subs:,} subs | {views:,} views")
+            else:
+                if idx % 15 == 0:
+                    print(f"[{idx}/{len(mappings)}] Processing {artist}...")
+        except Exception:
+            pass
+
+        time.sleep(0.4)
+
+    conn.commit()
+    conn.close()
+    print(f"\nLive scrape complete. Updated {success} / {len(mappings)} artists with real YouTube metrics.")
+
+
+if __name__ == "__main__":
+    main()