feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort

scripts/resolve_and_scrape_remaining_channels.py

@@ -0,0 +1,145 @@
+"""
+scripts/resolve_and_scrape_remaining_channels.py
+Automatically discovers canonical channel IDs and extracts live subscriber
+and view telemetry for any cohort artist currently unverified.
+"""
+
+import json
+import re
+import sys
+import time
+import urllib.parse
+from pathlib import Path
+import requests
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from core.database import get_connection
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+
+def parse_metric(raw_text):
+    if not raw_text:
+        return 0
+    t = str(raw_text).upper().replace(",", "").strip()
+    match = re.search(r"([\d\.]+)\s*([KMB])?", t)
+    if not match:
+        return 0
+    val = float(match.group(1))
+    suffix = match.group(2)
+    if suffix == "K":
+        val *= 1e3
+    elif suffix == "M":
+        val *= 1e6
+    elif suffix == "B":
+        val *= 1e9
+    return int(val)
+
+
+def extract_yt_initial_data(html_text):
+    match = re.search(r"var ytInitialData\s*=\s*({.+?});</script>", html_text)
+    if not match:
+        match = re.search(r"window\[\"ytInitialData\"\]\s*=\s*({.+?});", html_text)
+    if match:
+        try:
+            return json.loads(match.group(1))
+        except json.JSONDecodeError:
+            pass
+    return None
+
+
+def search_channel(artist_name):
+    query = f"{artist_name} official channel"
+    url = f"https://www.youtube.com/results?search_query={urllib.parse.quote_plus(query)}"
+    try:
+        r = requests.get(url, headers=HEADERS, timeout=12)
+        if r.status_code != 200:
+            return None
+
+        data = extract_yt_initial_data(r.text)
+        if not data:
+            return None
+
+        # Traverse contents to find channelRenderer
+        contents = (
+            data.get("contents", {})
+            .get("twoColumnSearchResultsRenderer", {})
+            .get("primaryContents", {})
+            .get("sectionListRenderer", {})
+            .get("contents", [])
+        )
+
+        for sec in contents:
+            item_section = sec.get("itemSectionRenderer", {}).get("contents", [])
+            for item in item_section:
+                if "channelRenderer" in item:
+                    cr = item["channelRenderer"]
+                    sub_text = cr.get("subscriberCountText", {}).get("simpleText", "")
+                    if not sub_text:
+                        runs = cr.get("subscriberCountText", {}).get("runs", [])
+                        sub_text = "".join(r.get("text", "") for r in runs)
+
+                    channel_id = cr.get("channelId")
+                    subs = parse_metric(sub_text)
+                    if subs > 0:
+                        return {"channel_id": channel_id, "subscribers": subs}
+
+    except Exception:
+        pass
+    return None
+
+
+def main():
+    conn = get_connection()
+    cur = conn.cursor()
+
+    # Find cohort artists that need ground truth update
+    cur.execute("""
+        SELECT a.artist_name 
+        FROM artists a
+        JOIN telemetry_metrics t ON a.artist_id = t.artist_id
+        WHERE t.subscribers_count = 500000 OR t.subscribers_count IS NULL
+    """)
+    targets = [row[0] for row in cur.fetchall()]
+
+    print(f"Resolving verified channels for {len(targets)} artists...")
+    success = 0
+
+    for idx, artist in enumerate(targets, 1):
+        res = search_channel(artist)
+        if res and res["subscribers"] > 0:
+            success += 1
+            cur.execute(
+                """
+                UPDATE telemetry_metrics
+                SET subscribers_count = ?
+                WHERE artist_id IN (
+                    SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
+                )
+                """,
+                (res["subscribers"], artist),
+            )
+            print(f"[{idx}/{len(targets)}] {artist} -> {res['subscribers']:,} subs (ID: {res['channel_id']})")
+        else:
+            if idx % 10 == 0:
+                print(f"[{idx}/{len(targets)}] In progress: {artist}...")
+
+        time.sleep(0.5)
+
+    conn.commit()
+    conn.close()
+    print(f"\nCompleted: Verified and updated {success} / {len(targets)} channels.")
+
+
+if __name__ == "__main__":
+    main()