147 lines
6.0 KiB
Python
147 lines
6.0 KiB
Python
"""
|
|
collectors/spotify_fallback_collector.py
|
|
Permitted fallback collector for artists not indexed in Kworb top-5000 tables.
|
|
Extracts public streaming metadata via official Spotify embed/public OpenGraph headers.
|
|
Enforces strict provenance, date-stamping, and zero-imputation governance.
|
|
"""
|
|
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Dict, Optional
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from core.health_detector import ScraperCircuitBreaker
|
|
|
|
HEADERS = {
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
|
),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
"Connection": "close"
|
|
}
|
|
|
|
|
|
class SpotifyFallbackCollector:
|
|
"""
|
|
Fallback collector for artists absent from Kworb public tables.
|
|
Pulls verifiable metadata from public Spotify artist embed profiles.
|
|
"""
|
|
def __init__(self, circuit_breaker: Optional[ScraperCircuitBreaker] = None):
|
|
self.circuit_breaker = circuit_breaker or ScraperCircuitBreaker(failure_threshold=8, cooldown_seconds=5)
|
|
self.session = requests.Session()
|
|
self.session.headers.update(HEADERS)
|
|
|
|
def fetch_artist_fallback(
|
|
self,
|
|
artist_name: str,
|
|
spotify_id_or_uri: Optional[str] = None
|
|
) -> Dict[str, Optional[object]]:
|
|
"""
|
|
Attempts to resolve verified Spotify streaming telemetry via embed/public pages.
|
|
Returns a dictionary adhering to strict provenance logging.
|
|
"""
|
|
timestamp = datetime.now(timezone.utc).isoformat()
|
|
clean_name = artist_name.strip()
|
|
|
|
# If no verified artist ID is provided, do not speculate
|
|
if not spotify_id_or_uri or str(spotify_id_or_uri).strip().lower() in ("nan", "none", ""):
|
|
return {
|
|
"name": clean_name,
|
|
"spotify_id": None,
|
|
"spotify_monthly_listeners": None,
|
|
"spotify_daily_change": None,
|
|
"spotify_data_source": None,
|
|
"spotify_status": "UNAVAILABLE_NO_ID",
|
|
"collection_date": timestamp
|
|
}
|
|
|
|
clean_id = str(spotify_id_or_uri).replace("spotify:artist:", "").strip()
|
|
|
|
if self.circuit_breaker.is_tripped():
|
|
return {
|
|
"name": clean_name,
|
|
"spotify_id": clean_id,
|
|
"spotify_monthly_listeners": None,
|
|
"spotify_daily_change": None,
|
|
"spotify_data_source": None,
|
|
"spotify_status": "CIRCUIT_BREAKER_ACTIVE",
|
|
"collection_date": timestamp
|
|
}
|
|
|
|
url = f"https://open.spotify.com/embed/artist/{clean_id}"
|
|
try:
|
|
res = self.session.get(url, timeout=6)
|
|
if res.status_code == 200:
|
|
self.circuit_breaker.record_success()
|
|
soup = BeautifulSoup(res.text, "html.parser")
|
|
script_tag = soup.find("script", id="__NEXT_DATA__")
|
|
if script_tag and script_tag.string:
|
|
try:
|
|
data = json.loads(script_tag.string)
|
|
artist_entity = data.get("props", {}).get("pageProps", {}).get("state", {}).get("data", {}).get("artist", {})
|
|
entity_name = artist_entity.get("name", "")
|
|
|
|
if entity_name and (clean_name.lower() in entity_name.lower() or entity_name.lower() in clean_name.lower()):
|
|
listeners = artist_entity.get("monthly_listeners")
|
|
if listeners is not None:
|
|
return {
|
|
"name": clean_name,
|
|
"spotify_id": clean_id,
|
|
"spotify_monthly_listeners": int(listeners),
|
|
"spotify_daily_change": None,
|
|
"spotify_data_source": "spotify_embed_public",
|
|
"spotify_status": "VERIFIED_SPOTIFY_EMBED",
|
|
"collection_date": timestamp
|
|
}
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"name": clean_name,
|
|
"spotify_id": clean_id,
|
|
"spotify_monthly_listeners": None,
|
|
"spotify_daily_change": None,
|
|
"spotify_data_source": "spotify_embed_public",
|
|
"spotify_status": "UNAVAILABLE_UNLISTED_ON_EMBED",
|
|
"collection_date": timestamp
|
|
}
|
|
elif res.status_code in (404, 400):
|
|
return {
|
|
"name": clean_name,
|
|
"spotify_id": clean_id,
|
|
"spotify_monthly_listeners": None,
|
|
"spotify_daily_change": None,
|
|
"spotify_data_source": None,
|
|
"spotify_status": "NOT_FOUND",
|
|
"collection_date": timestamp
|
|
}
|
|
else:
|
|
return {
|
|
"name": clean_name,
|
|
"spotify_id": clean_id,
|
|
"spotify_monthly_listeners": None,
|
|
"spotify_daily_change": None,
|
|
"spotify_data_source": None,
|
|
"spotify_status": f"HTTP_ERROR_{res.status_code}",
|
|
"collection_date": timestamp
|
|
}
|
|
except Exception:
|
|
return {
|
|
"name": clean_name,
|
|
"spotify_id": clean_id,
|
|
"spotify_monthly_listeners": None,
|
|
"spotify_daily_change": None,
|
|
"spotify_data_source": None,
|
|
"spotify_status": "FETCH_EXCEPTION",
|
|
"collection_date": timestamp
|
|
} |