feat: complete Day 2 handoff with API service, PM2 process daemons, and test suites

enrich_kworb_deep_sweep.py

@@ -0,0 +1,158 @@
+"""
+enrich_kworb_deep_sweep.py
+Extended streaming telemetry ingestion across Kworb tables 1-4.
+Extracts verified Spotify Monthly Listeners and Daily Change for the HIFFI cohort.
+"""
+
+from datetime import datetime, timezone
+from pathlib import Path
+import re
+import time
+from bs4 import BeautifulSoup
+import pandas as pd
+import requests
+
+DATA_PATH = Path("data/hiffi_multiplatform_500.csv")
+
+KWORB_URLS = [
+    "https://kworb.net/spotify/listeners.html",
+    "https://kworb.net/spotify/listeners2.html",
+    "https://kworb.net/spotify/listeners3.html",
+    "https://kworb.net/spotify/listeners4.html",
+]
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+    )
+}
+
+STAGE_ALIASES = {
+    "a$ap rocky": "asap rocky",
+    "a$ap ferg": "asap ferg",
+    "a$ap mob": "asap mob",
+    "joey bada$$": "joey badass",
+    "6ix9ine": "sixnine",
+    "curren$y": "currensy",
+    "ty dolla $ign": "ty dolla sign",
+    "kool g rap": "kool g rap",
+    "t-pain": "tpain",
+    "jay-z": "jay z",
+    "e-40": "e40",
+    "yg": "yg",
+    "ti": "ti",
+    "t.i.": "ti",
+}
+
+
+def normalize_name(name: str) -> str:
+    if not name:
+        return ""
+    norm = str(name).strip().lower()
+    if norm in STAGE_ALIASES:
+        norm = STAGE_ALIASES[norm]
+    norm = norm.replace("$", "s").replace("&", "and")
+    norm = re.sub(r"[^\w\s]", "", norm)
+    return re.sub(r"\s+", " ", norm).strip()
+
+
+def build_kworb_cache() -> dict:
+    """Scrapes Kworb pages 1 to 4 and indexes all global artists."""
+    print(f"Scraping {len(KWORB_URLS)} Kworb global listener tables...")
+    cache = {}
+
+    for url in KWORB_URLS:
+        print(f"  Fetching: {url} ...", end="", flush=True)
+        try:
+            res = requests.get(url, headers=HEADERS, timeout=15)
+            if res.status_code != 200:
+                print(f" HTTP {res.status_code}")
+                continue
+
+            soup = BeautifulSoup(res.text, "html.parser")
+            rows = soup.find_all("tr")
+            parsed_on_page = 0
+
+            for row in rows:
+                cols = row.find_all("td")
+                if len(cols) >= 4:
+                    raw_artist = cols[1].get_text(strip=True)
+                    artist_norm = normalize_name(raw_artist)
+
+                    # Extract listeners
+                    raw_listeners = cols[2].get_text(strip=True).replace(",", "")
+                    try:
+                        listeners = int(raw_listeners)
+                    except ValueError:
+                        continue
+
+                    # Extract daily +/- change
+                    raw_daily = cols[3].get_text(strip=True).replace(",", "").replace("+", "")
+                    try:
+                        daily_change = int(raw_daily)
+                    except ValueError:
+                        daily_change = None
+
+                    if artist_norm and artist_norm not in cache:
+                        cache[artist_norm] = (listeners, daily_change)
+                        parsed_on_page += 1
+
+            print(f" Indexed {parsed_on_page} artists.")
+            time.sleep(1.0)
+
+        except Exception as e:
+            print(f" Error: {e}")
+
+    print(f"Total Kworb Cache Size: {len(cache):,} global artists.\n")
+    return cache
+
+
+def run_sweep():
+    if not DATA_PATH.exists():
+        print(f"Missing dataset: {DATA_PATH}")
+        return
+
+    df = pd.read_csv(DATA_PATH)
+    initial_valid = df["spotify_monthly_listeners"].notna().sum()
+    print(f"Master Cohort: {len(df)} artists | Current Valid Spotify: {initial_valid}")
+
+    kworb_cache = build_kworb_cache()
+    if not kworb_cache:
+        print("Kworb cache empty. Exiting.")
+        return
+
+    matched = 0
+    now_utc = datetime.now(timezone.utc).isoformat()
+
+    for idx, row in df.iterrows():
+        current_val = row.get("spotify_monthly_listeners")
+        if pd.notna(current_val) and current_val > 0:
+            continue
+
+        name = row["name"]
+        norm_key = normalize_name(name)
+
+        if norm_key in kworb_cache:
+            listeners, daily = kworb_cache[norm_key]
+            df.at[idx, "spotify_monthly_listeners"] = listeners
+            if daily is not None:
+                df.at[idx, "spotify_daily_change"] = daily
+            df.at[idx, "spotify_updated_at"] = now_utc
+            df.at[idx, "spotify_data_source"] = "kworb_listeners_deep_sweep"
+            matched += 1
+            print(f"  [MATCHED] {name} -> Listeners: {listeners:,} | Daily Change: {daily}")
+
+    df.to_csv(DATA_PATH, index=False)
+    final_valid = df["spotify_monthly_listeners"].notna().sum()
+
+    print("\n==========================================")
+    print("Kworb Sweep Completed!")
+    print(f"Newly Matched:  {matched}")
+    print(f"Total Coverage: {final_valid} / {len(df)} ({(final_valid/len(df))*100:.1f}%)")
+    print(f"Saved directly to: {DATA_PATH}")
+    print("==========================================")
+
+
+if __name__ == "__main__":
+    run_sweep()