feat: add Instagram candidate verifier, Spotify fallback collector, and expand QA suite to 12 tests

collectors/spotify_fallback_collector.py

@@ -0,0 +1,147 @@
+"""
+collectors/spotify_fallback_collector.py
+Permitted fallback collector for artists not indexed in Kworb top-5000 tables.
+Extracts public streaming metadata via official Spotify embed/public OpenGraph headers.
+Enforces strict provenance, date-stamping, and zero-imputation governance.
+"""
+
+from datetime import datetime, timezone
+import json
+import sys
+from pathlib import Path
+from typing import Dict, Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from core.health_detector import ScraperCircuitBreaker
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Connection": "close"
+}
+
+
+class SpotifyFallbackCollector:
+    """
+    Fallback collector for artists absent from Kworb public tables.
+    Pulls verifiable metadata from public Spotify artist embed profiles.
+    """
+    def __init__(self, circuit_breaker: Optional[ScraperCircuitBreaker] = None):
+        self.circuit_breaker = circuit_breaker or ScraperCircuitBreaker(failure_threshold=8, cooldown_seconds=5)
+        self.session = requests.Session()
+        self.session.headers.update(HEADERS)
+
+    def fetch_artist_fallback(
+        self,
+        artist_name: str,
+        spotify_id_or_uri: Optional[str] = None
+    ) -> Dict[str, Optional[object]]:
+        """
+        Attempts to resolve verified Spotify streaming telemetry via embed/public pages.
+        Returns a dictionary adhering to strict provenance logging.
+        """
+        timestamp = datetime.now(timezone.utc).isoformat()
+        clean_name = artist_name.strip()
+
+        # If no verified artist ID is provided, do not speculate
+        if not spotify_id_or_uri or str(spotify_id_or_uri).strip().lower() in ("nan", "none", ""):
+            return {
+                "name": clean_name,
+                "spotify_id": None,
+                "spotify_monthly_listeners": None,
+                "spotify_daily_change": None,
+                "spotify_data_source": None,
+                "spotify_status": "UNAVAILABLE_NO_ID",
+                "collection_date": timestamp
+            }
+
+        clean_id = str(spotify_id_or_uri).replace("spotify:artist:", "").strip()
+
+        if self.circuit_breaker.is_tripped():
+            return {
+                "name": clean_name,
+                "spotify_id": clean_id,
+                "spotify_monthly_listeners": None,
+                "spotify_daily_change": None,
+                "spotify_data_source": None,
+                "spotify_status": "CIRCUIT_BREAKER_ACTIVE",
+                "collection_date": timestamp
+            }
+
+        url = f"https://open.spotify.com/embed/artist/{clean_id}"
+        try:
+            res = self.session.get(url, timeout=6)
+            if res.status_code == 200:
+                self.circuit_breaker.record_success()
+                soup = BeautifulSoup(res.text, "html.parser")
+                script_tag = soup.find("script", id="__NEXT_DATA__")
+                if script_tag and script_tag.string:
+                    try:
+                        data = json.loads(script_tag.string)
+                        artist_entity = data.get("props", {}).get("pageProps", {}).get("state", {}).get("data", {}).get("artist", {})
+                        entity_name = artist_entity.get("name", "")
+                        
+                        if entity_name and (clean_name.lower() in entity_name.lower() or entity_name.lower() in clean_name.lower()):
+                            listeners = artist_entity.get("monthly_listeners")
+                            if listeners is not None:
+                                return {
+                                    "name": clean_name,
+                                    "spotify_id": clean_id,
+                                    "spotify_monthly_listeners": int(listeners),
+                                    "spotify_daily_change": None,
+                                    "spotify_data_source": "spotify_embed_public",
+                                    "spotify_status": "VERIFIED_SPOTIFY_EMBED",
+                                    "collection_date": timestamp
+                                }
+                    except Exception:
+                        pass
+
+                return {
+                    "name": clean_name,
+                    "spotify_id": clean_id,
+                    "spotify_monthly_listeners": None,
+                    "spotify_daily_change": None,
+                    "spotify_data_source": "spotify_embed_public",
+                    "spotify_status": "UNAVAILABLE_UNLISTED_ON_EMBED",
+                    "collection_date": timestamp
+                }
+            elif res.status_code in (404, 400):
+                return {
+                    "name": clean_name,
+                    "spotify_id": clean_id,
+                    "spotify_monthly_listeners": None,
+                    "spotify_daily_change": None,
+                    "spotify_data_source": None,
+                    "spotify_status": "NOT_FOUND",
+                    "collection_date": timestamp
+                }
+            else:
+                return {
+                    "name": clean_name,
+                    "spotify_id": clean_id,
+                    "spotify_monthly_listeners": None,
+                    "spotify_daily_change": None,
+                    "spotify_data_source": None,
+                    "spotify_status": f"HTTP_ERROR_{res.status_code}",
+                    "collection_date": timestamp
+                }
+        except Exception:
+            return {
+                "name": clean_name,
+                "spotify_id": clean_id,
+                "spotify_monthly_listeners": None,
+                "spotify_daily_change": None,
+                "spotify_data_source": None,
+                "spotify_status": "FETCH_EXCEPTION",
+                "collection_date": timestamp
+            }