feat: add Instagram candidate verifier, Spotify fallback collector, and expand QA suite to 12 tests

collectors/instagram_verifier.py

@@ -0,0 +1,219 @@
+"""
+collectors/instagram_verifier.py
+Automated candidate matching and verification engine for Instagram profiles.
+Replaces brittle static handle overrides with multi-signal evidence verification.
+"""
+
+from datetime import datetime, timezone
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json"
+
+
+def pd_isna(val) -> bool:
+    if val is None:
+        return True
+    if isinstance(val, float) and str(val) == "nan":
+        return True
+    return False
+
+
+class InstagramCandidateVerifier:
+    """
+    Evaluates Instagram account candidates using name matching, bio link analysis,
+    and follower plausibility heuristics. Emits confidence scores and audit evidence.
+    """
+    def __init__(self, overrides_path: Optional[Path] = None):
+        self.overrides_path = overrides_path or OVERRIDES_PATH
+        self.manual_overrides: Dict[str, object] = {}
+        self._load_overrides()
+
+    def _load_overrides(self):
+        if self.overrides_path.exists():
+            try:
+                with open(self.overrides_path, "r", encoding="utf-8") as f:
+                    self.manual_overrides = json.load(f)
+            except Exception as e:
+                print(f"[Verifier] Warning: Could not load overrides: {e}")
+
+    def clean_text(self, text: Optional[object]) -> str:
+        """Sanitizes text, safely guarding against None, NaN floats, and non-string types."""
+        if text is None or pd_isna(text):
+            return ""
+        text_str = str(text).strip()
+        if text_str.lower() in ("nan", "none", ""):
+            return ""
+        return re.sub(r"[^a-zA-Z0-9]", "", text_str).lower()
+
+    def extract_tokens(self, text: Optional[object]) -> List[str]:
+        """Extracts significant tokens of length >= 3 for fuzzy-token matching."""
+        if text is None or pd_isna(text):
+            return []
+        raw_words = re.findall(r"[a-zA-Z0-9]+", str(text).lower())
+        stop_words = {"the", "and", "official", "real", "music", "tv", "page"}
+        return [w for w in raw_words if len(w) >= 3 and w not in stop_words]
+
+    def evaluate_candidate(
+        self,
+        artist_name: str,
+        candidate_handle: Optional[str],
+        profile_name: Optional[str] = None,
+        bio_text: Optional[str] = None,
+        bio_links: Optional[List[str]] = None,
+        follower_count: Optional[int] = None,
+        youtube_channel_name: Optional[str] = None
+    ) -> Dict[str, object]:
+        """
+        Calculates verification status, confidence score [0.0, 1.0], and evidence trail.
+        """
+        timestamp = datetime.now(timezone.utc).isoformat()
+        clean_artist = str(artist_name).strip()
+        evidence: List[str] = []
+
+        # 1. Check Highest Priority Source: Manual Verified Overrides
+        for manual_name, data in self.manual_overrides.items():
+            if str(manual_name).strip().lower() == clean_artist.lower():
+                if isinstance(data, dict):
+                    m_handle = data.get("instagram_handle", candidate_handle)
+                    m_followers = data.get("instagram_followers", follower_count)
+                elif isinstance(data, str):
+                    m_handle = data
+                    m_followers = follower_count
+                else:
+                    m_handle = candidate_handle
+                    m_followers = follower_count
+
+                evidence.append(f"Matched manual curated override in {self.overrides_path.name}")
+                return {
+                    "artist_name": clean_artist,
+                    "handle": m_handle,
+                    "followers": m_followers,
+                    "confidence_score": 1.0,
+                    "verification_status": "MANUAL_VERIFIED",
+                    "evidence": "; ".join(evidence),
+                    "evaluation_date": timestamp
+                }
+
+        # 2. Check Candidate Handle Validity
+        if not candidate_handle or pd_isna(candidate_handle) or str(candidate_handle).strip().lower() in ("nan", "none", ""):
+            return {
+                "artist_name": clean_artist,
+                "handle": None,
+                "followers": None,
+                "confidence_score": 0.0,
+                "verification_status": "UNMATCHED_NO_HANDLE",
+                "evidence": "No candidate handle available",
+                "evaluation_date": timestamp
+            }
+
+        handle = str(candidate_handle).replace("@", "").strip().lower()
+        clean_handle = self.clean_text(handle)
+        clean_artist_norm = self.clean_text(clean_artist)
+
+        # 3. Disqualify Fan / Parody Patterns
+        fan_keywords = ["fanpage", "fanclub", "updates", "backup", "tribute", "parody", "daily"]
+        bio_raw = str(bio_text or "")
+        if any(kw in handle for kw in fan_keywords) or (bio_text and any(kw in bio_raw.lower() for kw in fan_keywords)):
+            evidence.append(f"Rejected: handle or bio contains fan/parody marker ({handle})")
+            return {
+                "artist_name": clean_artist,
+                "handle": handle,
+                "followers": None,
+                "confidence_score": 0.0,
+                "verification_status": "REJECTED_FAN_PAGE",
+                "evidence": "; ".join(evidence),
+                "evaluation_date": timestamp
+            }
+
+        # 4. Multi-Signal Confidence Scoring
+        confidence = 0.0
+
+        # Exact Alphanumeric Match
+        if clean_handle and clean_handle == clean_artist_norm:
+            confidence += 0.40
+            evidence.append("Exact alphanumeric match between artist name and handle")
+        # Substring Containment
+        elif clean_handle and clean_artist_norm and (clean_artist_norm in clean_handle or clean_handle in clean_artist_norm):
+            confidence += 0.30
+            evidence.append("Substring containment between artist name and handle")
+        else:
+            # Token Overlap Check
+            artist_tokens = self.extract_tokens(clean_artist)
+            handle_tokens = self.extract_tokens(handle)
+            overlap = [t for t in artist_tokens if t in clean_handle or any(t in ht for ht in handle_tokens)]
+            if overlap:
+                confidence += 0.25
+                evidence.append(f"Token overlap identified: {overlap}")
+            else:
+                confidence += 0.10
+                evidence.append("Candidate handle linked from cohort")
+
+        # Profile Display Name Alignment
+        clean_profile = self.clean_text(profile_name)
+        if clean_profile and clean_artist_norm:
+            if clean_artist_norm in clean_profile or clean_profile in clean_artist_norm:
+                confidence += 0.20
+                evidence.append(f"Display name '{profile_name}' aligns with artist name")
+
+        # YouTube Channel Alignment
+        clean_yt = self.clean_text(youtube_channel_name)
+        if clean_yt and clean_handle:
+            if clean_handle in clean_yt or clean_yt in clean_handle:
+                confidence += 0.15
+                evidence.append(f"Handle correlates with YouTube channel '{youtube_channel_name}'")
+
+        # Cross-Linked External Proof (Linktree / Spotify / etc.)
+        links = bio_links or []
+        bio_lower = bio_raw.lower()
+        if any("linktr.ee" in l.lower() or "beacons.ai" in l.lower() or "spotify.com" in l.lower() for l in links) or \
+           any(domain in bio_lower for domain in ["linktr.ee", "spotify.com", "music.apple.com", "official"]):
+            confidence += 0.25
+            evidence.append("Bio contains cross-platform music or Linktree link")
+
+        # Follower Count Credibility Gate
+        has_valid_followers = False
+        if follower_count is not None and not pd_isna(follower_count):
+            try:
+                f_count = int(follower_count)
+                if f_count < 10000:
+                    confidence = min(confidence, 0.40)
+                    evidence.append(f"Follower count ({f_count:,}) is below 10k threshold; quarantined")
+                else:
+                    confidence += 0.35
+                    has_valid_followers = True
+                    evidence.append(f"Follower volume ({f_count:,}) meets verified public artist tier")
+            except (ValueError, TypeError):
+                pass
+        else:
+            evidence.append("No follower count provided to assess volume")
+
+        confidence = round(min(confidence, 1.0), 2)
+
+        # 5. Determine Verification Status
+        if confidence >= 0.55 or (has_valid_followers and confidence >= 0.40):
+            status = "AUTOMATED_VERIFIED"
+        elif confidence >= 0.25:
+            status = "UNCERTAIN_NEEDS_REVIEW"
+        else:
+            status = "REJECTED_LOW_CONFIDENCE"
+
+        # Preserve follower signals for both verified and review accounts; only strip on rejection
+        retained_followers = follower_count if status in ("MANUAL_VERIFIED", "AUTOMATED_VERIFIED", "UNCERTAIN_NEEDS_REVIEW") else None
+
+        return {
+            "artist_name": clean_artist,
+            "handle": handle if status != "REJECTED_LOW_CONFIDENCE" else None,
+            "followers": retained_followers,
+            "confidence_score": confidence,
+            "verification_status": status,
+            "evidence": "; ".join(evidence),
+            "evaluation_date": timestamp
+        }

collectors/spotify_fallback_collector.py

@@ -0,0 +1,147 @@
+"""
+collectors/spotify_fallback_collector.py
+Permitted fallback collector for artists not indexed in Kworb top-5000 tables.
+Extracts public streaming metadata via official Spotify embed/public OpenGraph headers.
+Enforces strict provenance, date-stamping, and zero-imputation governance.
+"""
+
+from datetime import datetime, timezone
+import json
+import sys
+from pathlib import Path
+from typing import Dict, Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from core.health_detector import ScraperCircuitBreaker
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Connection": "close"
+}
+
+
+class SpotifyFallbackCollector:
+    """
+    Fallback collector for artists absent from Kworb public tables.
+    Pulls verifiable metadata from public Spotify artist embed profiles.
+    """
+    def __init__(self, circuit_breaker: Optional[ScraperCircuitBreaker] = None):
+        self.circuit_breaker = circuit_breaker or ScraperCircuitBreaker(failure_threshold=8, cooldown_seconds=5)
+        self.session = requests.Session()
+        self.session.headers.update(HEADERS)
+
+    def fetch_artist_fallback(
+        self,
+        artist_name: str,
+        spotify_id_or_uri: Optional[str] = None
+    ) -> Dict[str, Optional[object]]:
+        """
+        Attempts to resolve verified Spotify streaming telemetry via embed/public pages.
+        Returns a dictionary adhering to strict provenance logging.
+        """
+        timestamp = datetime.now(timezone.utc).isoformat()
+        clean_name = artist_name.strip()
+
+        # If no verified artist ID is provided, do not speculate
+        if not spotify_id_or_uri or str(spotify_id_or_uri).strip().lower() in ("nan", "none", ""):
+            return {
+                "name": clean_name,
+                "spotify_id": None,
+                "spotify_monthly_listeners": None,
+                "spotify_daily_change": None,
+                "spotify_data_source": None,
+                "spotify_status": "UNAVAILABLE_NO_ID",
+                "collection_date": timestamp
+            }
+
+        clean_id = str(spotify_id_or_uri).replace("spotify:artist:", "").strip()
+
+        if self.circuit_breaker.is_tripped():
+            return {
+                "name": clean_name,
+                "spotify_id": clean_id,
+                "spotify_monthly_listeners": None,
+                "spotify_daily_change": None,
+                "spotify_data_source": None,
+                "spotify_status": "CIRCUIT_BREAKER_ACTIVE",
+                "collection_date": timestamp
+            }
+
+        url = f"https://open.spotify.com/embed/artist/{clean_id}"
+        try:
+            res = self.session.get(url, timeout=6)
+            if res.status_code == 200:
+                self.circuit_breaker.record_success()
+                soup = BeautifulSoup(res.text, "html.parser")
+                script_tag = soup.find("script", id="__NEXT_DATA__")
+                if script_tag and script_tag.string:
+                    try:
+                        data = json.loads(script_tag.string)
+                        artist_entity = data.get("props", {}).get("pageProps", {}).get("state", {}).get("data", {}).get("artist", {})
+                        entity_name = artist_entity.get("name", "")
+                        
+                        if entity_name and (clean_name.lower() in entity_name.lower() or entity_name.lower() in clean_name.lower()):
+                            listeners = artist_entity.get("monthly_listeners")
+                            if listeners is not None:
+                                return {
+                                    "name": clean_name,
+                                    "spotify_id": clean_id,
+                                    "spotify_monthly_listeners": int(listeners),
+                                    "spotify_daily_change": None,
+                                    "spotify_data_source": "spotify_embed_public",
+                                    "spotify_status": "VERIFIED_SPOTIFY_EMBED",
+                                    "collection_date": timestamp
+                                }
+                    except Exception:
+                        pass
+
+                return {
+                    "name": clean_name,
+                    "spotify_id": clean_id,
+                    "spotify_monthly_listeners": None,
+                    "spotify_daily_change": None,
+                    "spotify_data_source": "spotify_embed_public",
+                    "spotify_status": "UNAVAILABLE_UNLISTED_ON_EMBED",
+                    "collection_date": timestamp
+                }
+            elif res.status_code in (404, 400):
+                return {
+                    "name": clean_name,
+                    "spotify_id": clean_id,
+                    "spotify_monthly_listeners": None,
+                    "spotify_daily_change": None,
+                    "spotify_data_source": None,
+                    "spotify_status": "NOT_FOUND",
+                    "collection_date": timestamp
+                }
+            else:
+                return {
+                    "name": clean_name,
+                    "spotify_id": clean_id,
+                    "spotify_monthly_listeners": None,
+                    "spotify_daily_change": None,
+                    "spotify_data_source": None,
+                    "spotify_status": f"HTTP_ERROR_{res.status_code}",
+                    "collection_date": timestamp
+                }
+        except Exception:
+            return {
+                "name": clean_name,
+                "spotify_id": clean_id,
+                "spotify_monthly_listeners": None,
+                "spotify_daily_change": None,
+                "spotify_data_source": None,
+                "spotify_status": "FETCH_EXCEPTION",
+                "collection_date": timestamp
+            }