feat: integrate daily velocity ingestion, circuit breaker, clean requirements, and resilient batch loader

collectors/kworb_spotify_collector.py

@@ -1,13 +1,13 @@
 """
 collectors/kworb_spotify_collector.py
-Audited alternative streaming telemetry collector.
-Parses verified public Spotify Monthly Listeners from Kworb.net aggregator tables.
-Enforces strict provenance attribution.
+Audited public streaming telemetry collector.
+Extracts verified Spotify Monthly Listeners and Daily Change (+/-) from Kworb.net tables.
+Enforces strict null governance (unlisted artists remain None/NaN).
 """
 
 import requests
 from bs4 import BeautifulSoup
-from typing import Dict, Optional
+from typing import Dict, Optional, Tuple
 
 KWORB_LISTENERS_URLS = [
     "https://kworb.net/spotify/listeners.html",
@@ -24,11 +24,12 @@ HEADERS = {
 
 class KworbStreamingCollector:
     def __init__(self):
-        self.cached_listeners: Dict[str, int] = {}
+        # Maps lowercase artist_name -> (monthly_listeners, daily_change)
+        self.cached_metrics: Dict[str, Tuple[int, Optional[int]]] = {}
         self.is_loaded = False
 
     def load_aggregator_tables(self, max_pages: int = 2):
-        """Scrapes and indexes artist listener counts from public HTML tables using BeautifulSoup."""
+        """Scrapes and indexes artist listener counts and daily velocity from public tables."""
         print(f"[KworbCollector] Indexing public streaming metrics across {max_pages} tables...")
         for url in KWORB_LISTENERS_URLS[:max_pages]:
             try:
@@ -38,37 +39,57 @@ class KworbStreamingCollector:
                     rows = soup.find_all("tr")
                     for row in rows:
                         cols = row.find_all("td")
-                        if len(cols) >= 3:
-                            # Col 0: Rank, Col 1: Artist (nested inside div > a), Col 2: Monthly Listeners
-                            artist_cell = cols[1]
-                            artist_name = artist_cell.get_text(strip=True).lower()
+                        if len(cols) >= 4:
+                            # Col 1: Artist
+                            artist_name = cols[1].get_text(strip=True).lower()
+
+                            # Col 2: Monthly Listeners
                             raw_listeners = cols[2].get_text(strip=True).replace(",", "")
                             try:
                                 listeners = int(raw_listeners)
-                                self.cached_listeners[artist_name] = listeners
                             except ValueError:
                                 continue
+
+                            # Col 3: Daily +/- Change
+                            raw_daily = cols[3].get_text(strip=True).replace(",", "").replace("+", "")
+                            try:
+                                daily_change = int(raw_daily)
+                            except ValueError:
+                                daily_change = None
+
+                            self.cached_metrics[artist_name] = (listeners, daily_change)
             except Exception as e:
                 print(f"[KworbCollector] Error reading {url}: {e}")
                 continue
 
         self.is_loaded = True
-        print(f"[KworbCollector] Loaded {len(self.cached_listeners):,} artist records into memory.")
+        print(f"[KworbCollector] Loaded {len(self.cached_metrics):,} artist telemetry records.")
 
-    def get_monthly_listeners(self, artist_name: str) -> Optional[int]:
+    def get_artist_metrics(self, artist_name: str) -> Tuple[Optional[int], Optional[int]]:
+        """Returns (monthly_listeners, daily_change) or (None, None)."""
         if not self.is_loaded:
             self.load_aggregator_tables(max_pages=2)
-        return self.cached_listeners.get(artist_name.strip().lower())
+        return self.cached_metrics.get(artist_name.strip().lower(), (None, None))
+
+    def get_monthly_listeners(self, artist_name: str) -> Optional[int]:
+        return self.get_artist_metrics(artist_name)[0]
+
+    def get_daily_change(self, artist_name: str) -> Optional[int]:
+        return self.get_artist_metrics(artist_name)[1]
 
 
 if __name__ == "__main__":
     collector = KworbStreamingCollector()
+    collector.load_aggregator_tables(max_pages=2)
     test_cohort = [
-        "Drake", "Kendrick Lamar", "SZA", "Future", "Latto", 
-        "Russ Millions", "Cash Cobain", "Larry June", "Big Boogie", "Rylo Rodriguez"
+        "Drake", "Kendrick Lamar", "SZA", "Future", 
+        "Cardi B", "Russ Millions", "Larry June", "Rylo Rodriguez"
     ]
-    print("\nTesting Kworb Streaming Telemetry Extraction:")
+    print("\nValidating Kworb Stream & Daily Velocity Extraction:")
+    print(f"{'Artist':20} | {'Monthly Listeners':>18} | {'Daily +/- Trend':>16}")
+    print("-" * 60)
     for artist in test_cohort:
-        listeners = collector.get_monthly_listeners(artist)
-        formatted = f"{listeners:,}" if listeners else "Not Listed (< Top 5000)"
-        print(f"  {artist:20} -> {formatted}")
+        listeners, daily = collector.get_artist_metrics(artist)
+        l_str = f"{listeners:,}" if listeners is not None else "Not in Top 5000"
+        d_str = f"{daily:+,}" if daily is not None else "N/A"
+        print(f"{artist:20} | {l_str:>18} | {d_str:>16}")