feat: complete multiplatform pipeline hardening, neutral anchor scoring, and automated health checks

collectors/kworb_spotify_collector.py

@@ -0,0 +1,74 @@
+"""
+collectors/kworb_spotify_collector.py
+Audited alternative streaming telemetry collector.
+Parses verified public Spotify Monthly Listeners from Kworb.net aggregator tables.
+Enforces strict provenance attribution.
+"""
+
+import requests
+from bs4 import BeautifulSoup
+from typing import Dict, Optional
+
+KWORB_LISTENERS_URLS = [
+    "https://kworb.net/spotify/listeners.html",
+    "https://kworb.net/spotify/listeners2.html"
+]
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+    )
+}
+
+
+class KworbStreamingCollector:
+    def __init__(self):
+        self.cached_listeners: Dict[str, int] = {}
+        self.is_loaded = False
+
+    def load_aggregator_tables(self, max_pages: int = 2):
+        """Scrapes and indexes artist listener counts from public HTML tables using BeautifulSoup."""
+        print(f"[KworbCollector] Indexing public streaming metrics across {max_pages} tables...")
+        for url in KWORB_LISTENERS_URLS[:max_pages]:
+            try:
+                res = requests.get(url, headers=HEADERS, timeout=12)
+                if res.status_code == 200:
+                    soup = BeautifulSoup(res.text, "html.parser")
+                    rows = soup.find_all("tr")
+                    for row in rows:
+                        cols = row.find_all("td")
+                        if len(cols) >= 3:
+                            # Col 0: Rank, Col 1: Artist (nested inside div > a), Col 2: Monthly Listeners
+                            artist_cell = cols[1]
+                            artist_name = artist_cell.get_text(strip=True).lower()
+                            raw_listeners = cols[2].get_text(strip=True).replace(",", "")
+                            try:
+                                listeners = int(raw_listeners)
+                                self.cached_listeners[artist_name] = listeners
+                            except ValueError:
+                                continue
+            except Exception as e:
+                print(f"[KworbCollector] Error reading {url}: {e}")
+                continue
+
+        self.is_loaded = True
+        print(f"[KworbCollector] Loaded {len(self.cached_listeners):,} artist records into memory.")
+
+    def get_monthly_listeners(self, artist_name: str) -> Optional[int]:
+        if not self.is_loaded:
+            self.load_aggregator_tables(max_pages=2)
+        return self.cached_listeners.get(artist_name.strip().lower())
+
+
+if __name__ == "__main__":
+    collector = KworbStreamingCollector()
+    test_cohort = [
+        "Drake", "Kendrick Lamar", "SZA", "Future", "Latto", 
+        "Russ Millions", "Cash Cobain", "Larry June", "Big Boogie", "Rylo Rodriguez"
+    ]
+    print("\nTesting Kworb Streaming Telemetry Extraction:")
+    for artist in test_cohort:
+        listeners = collector.get_monthly_listeners(artist)
+        formatted = f"{listeners:,}" if listeners else "Not Listed (< Top 5000)"
+        print(f"  {artist:20} -> {formatted}")

collectors/social_fallback_collector.py

@@ -0,0 +1,71 @@
+"""
+collectors/social_fallback_collector.py
+Resolves missing Instagram handles via the official MusicBrainz REST API.
+Extracts verified url-rel claims without scraping search engines.
+"""
+
+import re
+import time
+import requests
+from typing import Optional, Dict, List
+
+MB_BASE_URL = "https://musicbrainz.org/ws/2/artist/"
+HEADERS = {
+    "User-Agent": "HIFFIArtistIntelligence/1.0 (sri@superlabs.ai)",
+    "Accept": "application/json"
+}
+
+def resolve_musicbrainz_instagram(artist_name: str) -> Optional[str]:
+    """
+    Searches MusicBrainz for the artist and inspects url-rels for Instagram profiles.
+    """
+    search_url = f"{MB_BASE_URL}?query=artist:{requests.utils.quote(artist_name)}&fmt=json&limit=3"
+    try:
+        res = requests.get(search_url, headers=HEADERS, timeout=8)
+        if res.status_code != 200:
+            return None
+        
+        artists = res.json().get("artists", [])
+        if not artists:
+            return None
+
+        # Check top matches
+        for artist in artists:
+            mbid = artist.get("id")
+            if not mbid:
+                continue
+
+            rel_url = f"{MB_BASE_URL}{mbid}?inc=url-rels&fmt=json"
+            time.sleep(1.0)  # MusicBrainz rate limit compliance: 1 req/sec
+            rel_res = requests.get(rel_url, headers=HEADERS, timeout=8)
+            if rel_res.status_code == 200:
+                relations = rel_res.json().get("relations", [])
+                for rel in relations:
+                    target = rel.get("url", {}).get("resource", "")
+                    if "instagram.com" in target:
+                        match = re.search(r"instagram\.com/([A-Za-z0-9_.]+)", target)
+                        if match:
+                            handle = match.group(1).lower().rstrip("/.")
+                            if handle not in {"p", "reel", "explore"}:
+                                return handle
+    except Exception:
+        return None
+
+    return None
+
+
+class SocialFallbackResolver:
+    def resolve_handles(self, artist_names: List[str]) -> Dict[str, Optional[str]]:
+        results = {}
+        for name in artist_names:
+            results[name] = resolve_musicbrainz_instagram(name)
+        return results
+
+
+if __name__ == "__main__":
+    resolver = SocialFallbackResolver()
+    test_cohort = ["Rylo Rodriguez", "Q Da Fool", "Flatbush Zombies"]
+    print("Testing MusicBrainz Entity Fallback Resolver:")
+    for artist in test_cohort:
+        h = resolve_musicbrainz_instagram(artist)
+        print(f"  {artist} -> @{h}")