hiffi-artist-intelligence/scripts/scrape_spotify_218.py

"""
scripts/scrape_spotify_218.py
Extracts verified Spotify followers and monthly listeners for the cohort artists
and writes ground truth directly into SQLite.
"""

import base64
import json
import os
import re
import sys
import time
from pathlib import Path
import pandas as pd
import requests
from dotenv import load_dotenv

ROOT_DIR = Path(__file__).resolve().parent.parent
if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

load_dotenv(ROOT_DIR / ".env")

from core.database import get_connection

CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")

HEADERS_WEB = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/124.0.0.0 Safari/537.36"
    ),
    "Accept-Language": "en-US,en;q=0.9",
}


def get_spotify_token():
    """Fetches an official application access token from Spotify Web API."""
    if not CLIENT_ID or not CLIENT_SECRET:
        return None
    auth_str = f"{CLIENT_ID}:{CLIENT_SECRET}"
    b64_auth = base64.b64encode(auth_str.encode()).decode()
    url = "https://accounts.spotify.com/api/token"
    headers = {"Authorization": f"Basic {b64_auth}"}
    data = {"grant_type": "client_credentials"}
    try:
        r = requests.post(url, headers=headers, data=data, timeout=10)
        if r.status_code == 200:
            return r.json().get("access_token")
    except Exception:
        pass
    return None


def search_spotify_artist(name, token):
    """Searches official Spotify API for canonical artist ID and follower count."""
    if not token:
        return None
    url = "https://api.spotify.com/v1/search"
    headers = {"Authorization": f"Bearer {token}"}
    params = {"q": name, "type": "artist", "limit": 1}
    try:
        r = requests.get(url, headers=headers, params=params, timeout=10)
        if r.status_code == 200:
            items = r.json().get("artists", {}).get("items", [])
            if items:
                artist = items[0]
                return {
                    "id": artist["id"],
                    "name": artist["name"],
                    "followers": artist.get("followers", {}).get("total", 0),
                    "popularity": artist.get("popularity", 0),
                }
    except Exception:
        pass
    return None


def scrape_monthly_listeners(spotify_id):
    """Parses monthly listeners from Spotify public artist profile page."""
    url = f"https://open.spotify.com/artist/{spotify_id}"
    try:
        r = requests.get(url, headers=HEADERS_WEB, timeout=10)
        if r.status_code == 200:
            # Match monthlyListeners in JSON hydration or state payload
            match_json = re.search(r'"monthlyListeners":\s*(\d+)', r.text)
            if match_json:
                return int(match_json.group(1))

            match_text = re.search(r'([\d,]+)\s+monthly\s+listeners', r.text, re.IGNORECASE)
            if match_text:
                return int(match_text.group(1).replace(",", ""))
    except Exception:
        pass
    return 0


def main():
    print("=" * 65)
    print("HIFFI INTELLIGENCE: SPOTIFY 218 LIVE EXTRACTION PIPELINE")
    print("=" * 65)

    token = get_spotify_token()
    if not token:
        print("[!] ERROR: SPOTIFY_CLIENT_ID or SPOTIFY_CLIENT_SECRET is missing or invalid in .env")
        return

    # Check for cohort CSV
    csv_path = ROOT_DIR / "data" / "hiffi_multiplatform_500.csv"
    if not csv_path.exists():
        csv_path = ROOT_DIR / "cleaned_artists_inventory.csv"

    df = pd.read_csv(csv_path)
    name_col = "name" if "name" in df.columns else "artist_name"
    cohort_names = [str(n).strip() for n in df[name_col].dropna()]

    conn = get_connection()
    cur = conn.cursor()

    # Identify artists missing verified monthly listeners in DB
    cur.execute("""
        SELECT a.artist_name 
        FROM artists a
        JOIN telemetry_metrics t ON a.artist_id = t.artist_id
        WHERE t.spotify_monthly_listeners IS NULL OR t.spotify_monthly_listeners = 0
    """)
    rows = cur.fetchall()
    missing_targets = [r[0] for r in rows if r[0] in cohort_names]

    if not missing_targets:
        print("[*] All artists already have verified monthly listeners. Checking cohort followers...")
        missing_targets = cohort_names[:50]

    print(f"Targeting {len(missing_targets)} artists for Spotify collection...\n")

    checkpoint_records = []
    success = 0

    for idx, artist_name in enumerate(missing_targets, 1):
        artist_meta = search_spotify_artist(artist_name, token)
        listeners = 0
        followers = 0
        sp_id = None

        if artist_meta:
            sp_id = artist_meta["id"]
            followers = artist_meta["followers"]
            listeners = scrape_monthly_listeners(sp_id)

        # Update DB if we resolved listeners or followers
        if listeners > 0 or followers > 0:
            success += 1
            if listeners > 0:
                cur.execute("""
                    UPDATE telemetry_metrics
                    SET spotify_monthly_listeners = ?
                    WHERE artist_id IN (
                        SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
                    )
                """, (listeners, artist_name))

            print(f"[{idx:>3}/{len(missing_targets)}] {artist_name:<25} -> {listeners:,} monthly listeners | {followers:,} followers")
        else:
            print(f"[{idx:>3}/{len(missing_targets)}] {artist_name:<25} -> Unresolved on Spotify")

        checkpoint_records.append({
            "name": artist_name,
            "spotify_id": sp_id,
            "spotify_monthly_listeners": listeners,
            "spotify_followers": followers,
        })

        time.sleep(0.3)

    conn.commit()
    conn.close()

    chk_dir = ROOT_DIR / "data" / "checkpoints"
    chk_dir.mkdir(parents=True, exist_ok=True)
    chk_path = chk_dir / "spotify_verification_checkpoint.csv"
    pd.DataFrame(checkpoint_records).to_csv(chk_path, index=False)

    print("\n" + "=" * 65)
    print(f"Finished. Extracted metrics for {success} / {len(missing_targets)} artists.")
    print(f"Checkpoint saved to: {chk_path}")
    print("=" * 65)


if __name__ == "__main__":
    main()