feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort

scripts/scrape_youtube_remaining_180.py

@@ -0,0 +1,195 @@
+"""
+scripts/scrape_youtube_remaining_180.py
+Extracts verified live YouTube subscriber counts for the remaining unverified artists
+by cross-referencing canonical channel identifiers from cleaned_artists_inventory.csv.
+"""
+
+import json
+import re
+import sys
+import time
+from pathlib import Path
+import pandas as pd
+import requests
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from core.database import get_connection
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+
+def parse_metric(raw_text):
+    if not raw_text:
+        return 0
+    t = str(raw_text).upper().replace(",", "").strip()
+    match = re.search(r"([\d\.]+)\s*([KMB])?", t)
+    if not match:
+        return 0
+    val = float(match.group(1))
+    suffix = match.group(2)
+    if suffix == "K":
+        val *= 1e3
+    elif suffix == "M":
+        val *= 1e6
+    elif suffix == "B":
+        val *= 1e9
+    return int(val)
+
+
+def extract_subscribers_from_html(html_text):
+    # Match ytInitialData embedded JSON
+    match = re.search(r"var ytInitialData\s*=\s*({.+?});</script>", html_text)
+    if not match:
+        match = re.search(r"window\[\"ytInitialData\"\]\s*=\s*({.+?});", html_text)
+
+    if match:
+        try:
+            data = json.loads(match.group(1))
+            header = data.get("header", {})
+
+            # Modern page header renderer
+            c4 = header.get("c4TabbedHeaderRenderer", {})
+            sub_text = c4.get("subscriberCountText", {}).get("simpleText", "")
+            if not sub_text:
+                runs = c4.get("subscriberCountText", {}).get("runs", [])
+                sub_text = "".join(r.get("text", "") for r in runs)
+
+            if sub_text:
+                return parse_metric(sub_text)
+
+            # Modern pageHeaderViewModel
+            vm = (
+                header.get("pageHeaderRenderer", {})
+                .get("content", {})
+                .get("pageHeaderViewModel", {})
+                .get("metadata", {})
+                .get("contentMetadataViewModel", {})
+                .get("metadataRows", [])
+            )
+            for row in vm:
+                for part in row.get("metadataParts", []):
+                    text = part.get("text", {}).get("content", "")
+                    if "subscribers" in text.lower():
+                        return parse_metric(text)
+        except Exception:
+            pass
+
+    # Direct fallback regex on raw HTML body
+    meta_match = re.search(r'([\d\.]+[KMB]?)\s+subscribers', html_text, re.IGNORECASE)
+    if meta_match:
+        return parse_metric(meta_match.group(1))
+
+    return 0
+
+
+def fetch_channel_stats(url_or_handle):
+    if not url_or_handle or pd.isna(url_or_handle):
+        return 0
+
+    url_str = str(url_or_handle).strip()
+    if not url_str.startswith("http"):
+        if url_str.startswith("UC"):
+            url_str = f"https://www.youtube.com/channel/{url_str}"
+        elif url_str.startswith("@"):
+            url_str = f"https://www.youtube.com/{url_str}"
+        else:
+            url_str = f"https://www.youtube.com/@{url_str}"
+
+    try:
+        r = requests.get(url_str, headers=HEADERS, timeout=12)
+        if r.status_code == 200:
+            return extract_subscribers_from_html(r.text)
+    except Exception:
+        pass
+    return 0
+
+
+def main():
+    print("=" * 65)
+    print("HIFFI INTELLIGENCE: YOUTUBE 180 CANONICAL EXTRACTION")
+    print("=" * 65)
+
+    # 1. Load canonical mapping from inventory
+    inv_file = ROOT_DIR / "cleaned_artists_inventory.csv"
+    inv_map = {}
+    if inv_file.exists():
+        inv_df = pd.read_csv(inv_file)
+        # Find column candidates for artist name and youtube url/channel
+        name_col = "name" if "name" in inv_df.columns else "artist_name"
+        yt_col = None
+        for col in ["youtube_url", "youtube", "channel_id", "youtube_channel", "social_youtube"]:
+            if col in inv_df.columns:
+                yt_col = col
+                break
+        if yt_col:
+            for _, row in inv_df.iterrows():
+                if pd.notna(row.get(name_col)) and pd.notna(row.get(yt_col)):
+                    inv_map[str(row[name_col]).strip().lower()] = str(row[yt_col]).strip()
+
+    # 2. Get list of artists needing verified scrape
+    enr_file = ROOT_DIR / "enriched_artists.csv"
+    enr_verified = set()
+    if enr_file.exists():
+        enr = pd.read_csv(enr_file)
+        enr_verified = set(
+            enr[enr["subscriber_count"].notna() & (enr["subscriber_count"] > 0)]["name"]
+            .str.strip()
+            .str.lower()
+        )
+
+    df500 = pd.read_csv(ROOT_DIR / "data" / "hiffi_multiplatform_500.csv")
+    cohort_names = [n.strip() for n in df500["name"]]
+    targets = [n for n in cohort_names if n.lower() not in enr_verified]
+
+    print(f"Targeting {len(targets)} artists for canonical YouTube resolution...")
+
+    conn = get_connection()
+    cur = conn.cursor()
+
+    updated = 0
+    for idx, name in enumerate(targets, 1):
+        target_url = inv_map.get(name.lower())
+        subs = 0
+
+        if target_url:
+            subs = fetch_channel_stats(target_url)
+
+        # Fallback to direct handle if inventory didn't have URL or returned 0
+        if subs == 0:
+            clean_handle = re.sub(r"[^a-zA-Z0-9]", "", name)
+            subs = fetch_channel_stats(f"https://www.youtube.com/@{clean_handle}")
+
+        if subs > 0:
+            updated += 1
+            cur.execute("""
+                UPDATE telemetry_metrics
+                SET subscribers_count = ?
+                WHERE artist_id IN (
+                    SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
+                )
+            """, (subs, name))
+            print(f"[{idx:>3}/{len(targets)}] {name:<25} -> {subs:,} subscribers")
+        else:
+            print(f"[{idx:>3}/{len(targets)}] {name:<25} -> Unresolved (Status: pending manual handle)")
+
+        time.sleep(0.4)
+
+    conn.commit()
+    conn.close()
+
+    print("=" * 65)
+    print(f"Finished. Successfully extracted and updated {updated} / {len(targets)} channels.")
+
+
+if __name__ == "__main__":
+    main()