hiffi-artist-intelligence/scripts/scrape_youtube_remaining_180.py

"""
scripts/scrape_youtube_remaining_180.py
Extracts verified live YouTube subscriber counts for the remaining unverified artists
by cross-referencing canonical channel identifiers from cleaned_artists_inventory.csv.
"""

import json
import re
import sys
import time
from pathlib import Path
import pandas as pd
import requests

ROOT_DIR = Path(__file__).resolve().parent.parent
if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

from core.database import get_connection

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/124.0.0.0 Safari/537.36"
    ),
    "Accept-Language": "en-US,en;q=0.9",
}


def parse_metric(raw_text):
    if not raw_text:
        return 0
    t = str(raw_text).upper().replace(",", "").strip()
    match = re.search(r"([\d\.]+)\s*([KMB])?", t)
    if not match:
        return 0
    val = float(match.group(1))
    suffix = match.group(2)
    if suffix == "K":
        val *= 1e3
    elif suffix == "M":
        val *= 1e6
    elif suffix == "B":
        val *= 1e9
    return int(val)


def extract_subscribers_from_html(html_text):
    # Match ytInitialData embedded JSON
    match = re.search(r"var ytInitialData\s*=\s*({.+?});</script>", html_text)
    if not match:
        match = re.search(r"window\[\"ytInitialData\"\]\s*=\s*({.+?});", html_text)

    if match:
        try:
            data = json.loads(match.group(1))
            header = data.get("header", {})

            # Modern page header renderer
            c4 = header.get("c4TabbedHeaderRenderer", {})
            sub_text = c4.get("subscriberCountText", {}).get("simpleText", "")
            if not sub_text:
                runs = c4.get("subscriberCountText", {}).get("runs", [])
                sub_text = "".join(r.get("text", "") for r in runs)

            if sub_text:
                return parse_metric(sub_text)

            # Modern pageHeaderViewModel
            vm = (
                header.get("pageHeaderRenderer", {})
                .get("content", {})
                .get("pageHeaderViewModel", {})
                .get("metadata", {})
                .get("contentMetadataViewModel", {})
                .get("metadataRows", [])
            )
            for row in vm:
                for part in row.get("metadataParts", []):
                    text = part.get("text", {}).get("content", "")
                    if "subscribers" in text.lower():
                        return parse_metric(text)
        except Exception:
            pass

    # Direct fallback regex on raw HTML body
    meta_match = re.search(r'([\d\.]+[KMB]?)\s+subscribers', html_text, re.IGNORECASE)
    if meta_match:
        return parse_metric(meta_match.group(1))

    return 0


def fetch_channel_stats(url_or_handle):
    if not url_or_handle or pd.isna(url_or_handle):
        return 0

    url_str = str(url_or_handle).strip()
    if not url_str.startswith("http"):
        if url_str.startswith("UC"):
            url_str = f"https://www.youtube.com/channel/{url_str}"
        elif url_str.startswith("@"):
            url_str = f"https://www.youtube.com/{url_str}"
        else:
            url_str = f"https://www.youtube.com/@{url_str}"

    try:
        r = requests.get(url_str, headers=HEADERS, timeout=12)
        if r.status_code == 200:
            return extract_subscribers_from_html(r.text)
    except Exception:
        pass
    return 0


def main():
    print("=" * 65)
    print("HIFFI INTELLIGENCE: YOUTUBE 180 CANONICAL EXTRACTION")
    print("=" * 65)

    # 1. Load canonical mapping from inventory
    inv_file = ROOT_DIR / "cleaned_artists_inventory.csv"
    inv_map = {}
    if inv_file.exists():
        inv_df = pd.read_csv(inv_file)
        # Find column candidates for artist name and youtube url/channel
        name_col = "name" if "name" in inv_df.columns else "artist_name"
        yt_col = None
        for col in ["youtube_url", "youtube", "channel_id", "youtube_channel", "social_youtube"]:
            if col in inv_df.columns:
                yt_col = col
                break
        if yt_col:
            for _, row in inv_df.iterrows():
                if pd.notna(row.get(name_col)) and pd.notna(row.get(yt_col)):
                    inv_map[str(row[name_col]).strip().lower()] = str(row[yt_col]).strip()

    # 2. Get list of artists needing verified scrape
    enr_file = ROOT_DIR / "enriched_artists.csv"
    enr_verified = set()
    if enr_file.exists():
        enr = pd.read_csv(enr_file)
        enr_verified = set(
            enr[enr["subscriber_count"].notna() & (enr["subscriber_count"] > 0)]["name"]
            .str.strip()
            .str.lower()
        )

    df500 = pd.read_csv(ROOT_DIR / "data" / "hiffi_multiplatform_500.csv")
    cohort_names = [n.strip() for n in df500["name"]]
    targets = [n for n in cohort_names if n.lower() not in enr_verified]

    print(f"Targeting {len(targets)} artists for canonical YouTube resolution...")

    conn = get_connection()
    cur = conn.cursor()

    updated = 0
    for idx, name in enumerate(targets, 1):
        target_url = inv_map.get(name.lower())
        subs = 0

        if target_url:
            subs = fetch_channel_stats(target_url)

        # Fallback to direct handle if inventory didn't have URL or returned 0
        if subs == 0:
            clean_handle = re.sub(r"[^a-zA-Z0-9]", "", name)
            subs = fetch_channel_stats(f"https://www.youtube.com/@{clean_handle}")

        if subs > 0:
            updated += 1
            cur.execute("""
                UPDATE telemetry_metrics
                SET subscribers_count = ?
                WHERE artist_id IN (
                    SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
                )
            """, (subs, name))
            print(f"[{idx:>3}/{len(targets)}] {name:<25} -> {subs:,} subscribers")
        else:
            print(f"[{idx:>3}/{len(targets)}] {name:<25} -> Unresolved (Status: pending manual handle)")

        time.sleep(0.4)

    conn.commit()
    conn.close()

    print("=" * 65)
    print(f"Finished. Successfully extracted and updated {updated} / {len(targets)} channels.")


if __name__ == "__main__":
    main()