hiffi-artist-intelligence/collectors/sync_remaining_spotify.py

"""
collectors/sync_remaining_spotify.py
Pulls verified monthly listener metrics for the 353 non-charting artists
via Spotify's public Embed & Partner JSON endpoints.
"""

import re
import time
import sqlite3
import logging
from pathlib import Path
from datetime import datetime, timezone
import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger("SpotifyTailSync")

PROJECT_ROOT = Path(__file__).resolve().parent.parent
DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"

HTTP_HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9"
}

def extract_spotify_id(url):
    m = re.search(r"artist/([0-9A-Za-z]{22})", str(url))
    return m.group(1) if m else None

def get_spotify_web_token(session):
    """Fetches an ephemeral anonymous web client bearer token directly from Spotify."""
    try:
        url = "https://open.spotify.com/get_access_token?reason=transport&productType=web_player"
        r = session.get(url, timeout=6)
        if r.status_code == 200:
            return r.json().get("accessToken")
    except Exception:
        pass
    return None

def fetch_monthly_listeners(session, sp_id, token=None):
    # Strategy 1: Public Embed JSON resource
    embed_url = f"https://open.spotify.com/embed/artist/{sp_id}"
    try:
        r = session.get(embed_url, timeout=6)
        if r.status_code == 200:
            # Pattern: "monthlyListeners":123456
            m = re.search(r'\"monthlyListeners\":\s*(\d+)', r.text)
            if m:
                return int(m.group(1))
            
            # Pattern: "123,456 monthly listeners"
            m2 = re.search(r'([\d,]+)\s+monthly listeners', r.text, re.IGNORECASE)
            if m2:
                return int(m2.group(1).replace(",", ""))
    except Exception:
        pass

    # Strategy 2: Anonymous partner GraphQL / metadata endpoint
    if token:
        try:
            api_url = f"https://api-partner.spotify.com/pathfinder/v1/query"
            params = {
                "operationName": "queryArtistOverview",
                "variables": f'{{"uris":["spotify:artist:{sp_id}"]}}',
                "extensions": '{"persistedQuery":{"version":1,"sha256Hash":"35ac4265af99890eac3ff8492314508c82a647970977821c4fa48956828551a3"}}'
            }
            h = dict(HTTP_HEADERS)
            h["Authorization"] = f"Bearer {token}"
            res = session.get(api_url, params=params, headers=h, timeout=6)
            if res.status_code == 200:
                data = res.json()
                count = (data.get("data", {})
                             .get("artists", [{}])[0]
                             .get("stats", {})
                             .get("monthlyListeners"))
                if count:
                    return int(count)
        except Exception:
            pass

    return None

def run():
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    # Artists who were not on Kworb's top 2,500 chart
    cur.execute("""
        SELECT a.artist_id, a.artist_name, a.spotify_url, t.spotify_monthly_listeners
        FROM artists a
        JOIN telemetry_metrics t ON a.artist_id = t.artist_id
        WHERE a.image_verified_flag = 1 
          AND (t.spotify_daily_change = 0 OR t.spotify_daily_change IS NULL)
    """)
    unresolved = cur.fetchall()
    logger.info(f"Targeting {len(unresolved)} remaining non-charting artists...")

    session = requests.Session()
    session.headers.update(HTTP_HEADERS)

    token = get_spotify_web_token(session)
    if token:
        logger.info("Acquired anonymous client access token for Spotify web service.")

    now_iso = datetime.now(timezone.utc).isoformat()
    updated = 0

    for idx, (artist_id, name, sp_url, current_val) in enumerate(unresolved, 1):
        sp_id = extract_spotify_id(sp_url)
        if not sp_id:
            continue

        listeners = fetch_monthly_listeners(session, sp_id, token)
        if listeners and listeners > 0:
            cur.execute("""
                UPDATE telemetry_metrics
                SET spotify_monthly_listeners = ?, last_synced_at = ?
                WHERE artist_id = ?
            """, (listeners, now_iso, artist_id))
            updated += 1
            logger.info(f"[{idx}/{len(unresolved)}] {name} -> {listeners:,} monthly listeners")
        else:
            logger.debug(f"[{idx}/{len(unresolved)}] Could not resolve {name}")

        if idx % 25 == 0:
            conn.commit()
            time.sleep(0.2)
        else:
            time.sleep(0.08)

    conn.commit()
    conn.close()
    logger.info(f"Tail sync finished. Updated {updated} / {len(unresolved)} artists.")

if __name__ == "__main__":
    run()