feat(pipeline): complete end-to-end multiplatform ingestion, QA governance, zero anomalies, and docs

collectors/spotify_api_collector.py

@@ -0,0 +1,179 @@
+"""
+collectors/spotify_api_collector.py
+Official Spotify Web API Collector utilizing OAuth 2.0 Client Credentials Flow.
+Provides authenticated telemetry fallback for artists unindexed on public aggregator tables.
+Adheres strictly to zero-imputation policy.
+"""
+
+import base64
+from datetime import datetime, timezone
+import os
+from pathlib import Path
+import sys
+import time
+from typing import Dict, Optional
+
+import requests
+from dotenv import load_dotenv
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from core.health_detector import ScraperCircuitBreaker
+
+# Load .env file if present
+load_dotenv(PROJECT_ROOT / ".env")
+
+
+class SpotifyAPICollector:
+    """
+    Handles token minting and artist queries via Spotify's Client Credentials flow.
+    Gracefully disables itself if credentials are not configured or access is restricted.
+    """
+    TOKEN_URL = "https://accounts.spotify.com/api/token"
+    ARTIST_URL = "https://api.spotify.com/v1/artists"
+
+    def __init__(
+        self,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        circuit_breaker: Optional[ScraperCircuitBreaker] = None
+    ):
+        raw_id = os.getenv("SPOTIFY_CLIENT_ID", "") if client_id is None else client_id
+        raw_secret = os.getenv("SPOTIFY_CLIENT_SECRET", "") if client_secret is None else client_secret
+
+        self.client_id = str(raw_id).strip() if raw_id else ""
+        self.client_secret = str(raw_secret).strip() if raw_secret else ""
+        self.circuit_breaker = circuit_breaker or ScraperCircuitBreaker(failure_threshold=3, cooldown_seconds=300)
+        self.access_token: Optional[str] = None
+        self.token_expiry: float = 0.0
+        self.is_configured = bool(self.client_id and self.client_secret)
+        self.is_access_forbidden = False
+
+    def _authenticate(self) -> bool:
+        """Exchanges client credentials for a bearer token."""
+        if not self.is_configured or self.is_access_forbidden:
+            return False
+
+        if self.access_token and time.time() < self.token_expiry - 60:
+            return True
+
+        if self.circuit_breaker.is_tripped():
+            return False
+
+        try:
+            creds = f"{self.client_id}:{self.client_secret}"
+            b64_creds = base64.b64encode(creds.encode("utf-8")).decode("utf-8")
+            headers = {
+                "Authorization": f"Basic {b64_creds}",
+                "Content-Type": "application/x-www-form-urlencoded"
+            }
+            data = {"grant_type": "client_credentials"}
+
+            resp = requests.post(self.TOKEN_URL, headers=headers, data=data, timeout=10)
+            if resp.status_code == 200:
+                payload = resp.json()
+                self.access_token = payload.get("access_token")
+                expires_in = payload.get("expires_in", 3600)
+                self.token_expiry = time.time() + expires_in
+                self.circuit_breaker.record_success()
+                return True
+            else:
+                self.circuit_breaker.record_failure(f"Auth HTTP {resp.status_code}")
+                return False
+        except Exception as e:
+            self.circuit_breaker.record_failure(f"Token exception: {e}")
+            return False
+
+    def fetch_artist(self, artist_id: str) -> Dict[str, object]:
+        """
+        Queries official artist details by Spotify ID/URI.
+        Zero-imputation: missing values return None.
+        Fast-fails immediately if circuit breaker is open or 403 Forbidden occurred.
+        """
+        now_utc = datetime.now(timezone.utc).isoformat()
+        clean_id = str(artist_id).replace("spotify:artist:", "").strip()
+
+        if not self.is_configured or not clean_id or clean_id.lower() in ("nan", "none", ""):
+            return {
+                "spotify_followers": None,
+                "spotify_popularity": None,
+                "spotify_data_source": None,
+                "spotify_status": "API_NOT_CONFIGURED" if not self.is_configured else "INVALID_ID",
+                "collection_date": now_utc
+            }
+
+        if self.is_access_forbidden or self.circuit_breaker.is_tripped():
+            return {
+                "spotify_followers": None,
+                "spotify_popularity": None,
+                "spotify_data_source": None,
+                "spotify_status": "API_CIRCUIT_OPEN",
+                "collection_date": now_utc
+            }
+
+        if not self._authenticate():
+            return {
+                "spotify_followers": None,
+                "spotify_popularity": None,
+                "spotify_data_source": None,
+                "spotify_status": "API_AUTH_FAILED",
+                "collection_date": now_utc
+            }
+
+        url = f"{self.ARTIST_URL}/{clean_id}"
+        headers = {"Authorization": f"Bearer {self.access_token}"}
+
+        try:
+            resp = requests.get(url, headers=headers, timeout=5)
+            if resp.status_code == 200:
+                data = resp.json()
+                followers = data.get("followers", {}).get("total")
+                popularity = data.get("popularity")
+                self.circuit_breaker.record_success()
+                return {
+                    "spotify_followers": followers,
+                    "spotify_popularity": popularity,
+                    "spotify_data_source": "official_spotify_web_api",
+                    "spotify_status": "VERIFIED_SPOTIFY_API",
+                    "collection_date": now_utc
+                }
+            elif resp.status_code == 403:
+                # Disable API tier gracefully so remaining cohort falls back to embed collector
+                self.is_access_forbidden = True
+                self.circuit_breaker.record_failure("API HTTP 403 - Restricted Access")
+                print("\n[SpotifyAPI] Notice: Spotify Web API returned HTTP 403. Disabling API tier and routing to Tier 3 embed fallback.")
+                return {
+                    "spotify_followers": None,
+                    "spotify_popularity": None,
+                    "spotify_data_source": None,
+                    "spotify_status": "API_ERROR_403",
+                    "collection_date": now_utc
+                }
+            elif resp.status_code == 404:
+                return {
+                    "spotify_followers": None,
+                    "spotify_popularity": None,
+                    "spotify_data_source": None,
+                    "spotify_status": "ARTIST_NOT_FOUND_ON_API",
+                    "collection_date": now_utc
+                }
+            else:
+                self.circuit_breaker.record_failure(f"API HTTP {resp.status_code}")
+                return {
+                    "spotify_followers": None,
+                    "spotify_popularity": None,
+                    "spotify_data_source": None,
+                    "spotify_status": f"API_ERROR_{resp.status_code}",
+                    "collection_date": now_utc
+                }
+        except Exception as e:
+            self.circuit_breaker.record_failure(f"API Request exception: {e}")
+            return {
+                "spotify_followers": None,
+                "spotify_popularity": None,
+                "spotify_data_source": None,
+                "spotify_status": "API_CONNECTION_ERROR",
+                "collection_date": now_utc
+            }