hiffi-artist-intelligence/collectors/spotify_fallback_collector.py

"""
collectors/spotify_fallback_collector.py
Permitted fallback collector for artists not indexed in Kworb top-5000 tables.
Extracts public streaming metadata via official Spotify embed/public OpenGraph headers.
Enforces strict provenance, date-stamping, and zero-imputation governance.
"""

from datetime import datetime, timezone
import json
import sys
from pathlib import Path
from typing import Dict, Optional

import requests
from bs4 import BeautifulSoup

PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from core.health_detector import ScraperCircuitBreaker

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    ),
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Connection": "close"
}


class SpotifyFallbackCollector:
    """
    Fallback collector for artists absent from Kworb public tables.
    Pulls verifiable metadata from public Spotify artist embed profiles.
    """
    def __init__(self, circuit_breaker: Optional[ScraperCircuitBreaker] = None):
        self.circuit_breaker = circuit_breaker or ScraperCircuitBreaker(failure_threshold=8, cooldown_seconds=5)
        self.session = requests.Session()
        self.session.headers.update(HEADERS)

    def fetch_artist_fallback(
        self,
        artist_name: str,
        spotify_id_or_uri: Optional[str] = None
    ) -> Dict[str, Optional[object]]:
        """
        Attempts to resolve verified Spotify streaming telemetry via embed/public pages.
        Returns a dictionary adhering to strict provenance logging.
        """
        timestamp = datetime.now(timezone.utc).isoformat()
        clean_name = artist_name.strip()

        # If no verified artist ID is provided, do not speculate
        if not spotify_id_or_uri or str(spotify_id_or_uri).strip().lower() in ("nan", "none", ""):
            return {
                "name": clean_name,
                "spotify_id": None,
                "spotify_monthly_listeners": None,
                "spotify_daily_change": None,
                "spotify_data_source": None,
                "spotify_status": "UNAVAILABLE_NO_ID",
                "collection_date": timestamp
            }

        clean_id = str(spotify_id_or_uri).replace("spotify:artist:", "").strip()

        if self.circuit_breaker.is_tripped():
            return {
                "name": clean_name,
                "spotify_id": clean_id,
                "spotify_monthly_listeners": None,
                "spotify_daily_change": None,
                "spotify_data_source": None,
                "spotify_status": "CIRCUIT_BREAKER_ACTIVE",
                "collection_date": timestamp
            }

        url = f"https://open.spotify.com/embed/artist/{clean_id}"
        try:
            res = self.session.get(url, timeout=6)
            if res.status_code == 200:
                self.circuit_breaker.record_success()
                soup = BeautifulSoup(res.text, "html.parser")
                script_tag = soup.find("script", id="__NEXT_DATA__")
                if script_tag and script_tag.string:
                    try:
                        data = json.loads(script_tag.string)
                        artist_entity = data.get("props", {}).get("pageProps", {}).get("state", {}).get("data", {}).get("artist", {})
                        entity_name = artist_entity.get("name", "")
                        
                        if entity_name and (clean_name.lower() in entity_name.lower() or entity_name.lower() in clean_name.lower()):
                            listeners = artist_entity.get("monthly_listeners")
                            if listeners is not None:
                                return {
                                    "name": clean_name,
                                    "spotify_id": clean_id,
                                    "spotify_monthly_listeners": int(listeners),
                                    "spotify_daily_change": None,
                                    "spotify_data_source": "spotify_embed_public",
                                    "spotify_status": "VERIFIED_SPOTIFY_EMBED",
                                    "collection_date": timestamp
                                }
                    except Exception:
                        pass

                return {
                    "name": clean_name,
                    "spotify_id": clean_id,
                    "spotify_monthly_listeners": None,
                    "spotify_daily_change": None,
                    "spotify_data_source": "spotify_embed_public",
                    "spotify_status": "UNAVAILABLE_UNLISTED_ON_EMBED",
                    "collection_date": timestamp
                }
            elif res.status_code in (404, 400):
                return {
                    "name": clean_name,
                    "spotify_id": clean_id,
                    "spotify_monthly_listeners": None,
                    "spotify_daily_change": None,
                    "spotify_data_source": None,
                    "spotify_status": "NOT_FOUND",
                    "collection_date": timestamp
                }
            else:
                return {
                    "name": clean_name,
                    "spotify_id": clean_id,
                    "spotify_monthly_listeners": None,
                    "spotify_daily_change": None,
                    "spotify_data_source": None,
                    "spotify_status": f"HTTP_ERROR_{res.status_code}",
                    "collection_date": timestamp
                }
        except Exception:
            return {
                "name": clean_name,
                "spotify_id": clean_id,
                "spotify_monthly_listeners": None,
                "spotify_daily_change": None,
                "spotify_data_source": None,
                "spotify_status": "FETCH_EXCEPTION",
                "collection_date": timestamp
            }