hiffi-artist-intelligence/enrich_kworb_deep_sweep.py

"""
enrich_kworb_deep_sweep.py
Extended streaming telemetry ingestion across Kworb tables 1-4.
Extracts verified Spotify Monthly Listeners and Daily Change for the HIFFI cohort.
"""

from datetime import datetime, timezone
from pathlib import Path
import re
import time
from bs4 import BeautifulSoup
import pandas as pd
import requests

DATA_PATH = Path("data/hiffi_multiplatform_500.csv")

KWORB_URLS = [
    "https://kworb.net/spotify/listeners.html",
    "https://kworb.net/spotify/listeners2.html",
    "https://kworb.net/spotify/listeners3.html",
    "https://kworb.net/spotify/listeners4.html",
]

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    )
}

STAGE_ALIASES = {
    "a$ap rocky": "asap rocky",
    "a$ap ferg": "asap ferg",
    "a$ap mob": "asap mob",
    "joey bada$$": "joey badass",
    "6ix9ine": "sixnine",
    "curren$y": "currensy",
    "ty dolla $ign": "ty dolla sign",
    "kool g rap": "kool g rap",
    "t-pain": "tpain",
    "jay-z": "jay z",
    "e-40": "e40",
    "yg": "yg",
    "ti": "ti",
    "t.i.": "ti",
}


def normalize_name(name: str) -> str:
    if not name:
        return ""
    norm = str(name).strip().lower()
    if norm in STAGE_ALIASES:
        norm = STAGE_ALIASES[norm]
    norm = norm.replace("$", "s").replace("&", "and")
    norm = re.sub(r"[^\w\s]", "", norm)
    return re.sub(r"\s+", " ", norm).strip()


def build_kworb_cache() -> dict:
    """Scrapes Kworb pages 1 to 4 and indexes all global artists."""
    print(f"Scraping {len(KWORB_URLS)} Kworb global listener tables...")
    cache = {}

    for url in KWORB_URLS:
        print(f"  Fetching: {url} ...", end="", flush=True)
        try:
            res = requests.get(url, headers=HEADERS, timeout=15)
            if res.status_code != 200:
                print(f" HTTP {res.status_code}")
                continue

            soup = BeautifulSoup(res.text, "html.parser")
            rows = soup.find_all("tr")
            parsed_on_page = 0

            for row in rows:
                cols = row.find_all("td")
                if len(cols) >= 4:
                    raw_artist = cols[1].get_text(strip=True)
                    artist_norm = normalize_name(raw_artist)

                    # Extract listeners
                    raw_listeners = cols[2].get_text(strip=True).replace(",", "")
                    try:
                        listeners = int(raw_listeners)
                    except ValueError:
                        continue

                    # Extract daily +/- change
                    raw_daily = cols[3].get_text(strip=True).replace(",", "").replace("+", "")
                    try:
                        daily_change = int(raw_daily)
                    except ValueError:
                        daily_change = None

                    if artist_norm and artist_norm not in cache:
                        cache[artist_norm] = (listeners, daily_change)
                        parsed_on_page += 1

            print(f" Indexed {parsed_on_page} artists.")
            time.sleep(1.0)

        except Exception as e:
            print(f" Error: {e}")

    print(f"Total Kworb Cache Size: {len(cache):,} global artists.\n")
    return cache


def run_sweep():
    if not DATA_PATH.exists():
        print(f"Missing dataset: {DATA_PATH}")
        return

    df = pd.read_csv(DATA_PATH)
    initial_valid = df["spotify_monthly_listeners"].notna().sum()
    print(f"Master Cohort: {len(df)} artists | Current Valid Spotify: {initial_valid}")

    kworb_cache = build_kworb_cache()
    if not kworb_cache:
        print("Kworb cache empty. Exiting.")
        return

    matched = 0
    now_utc = datetime.now(timezone.utc).isoformat()

    for idx, row in df.iterrows():
        current_val = row.get("spotify_monthly_listeners")
        if pd.notna(current_val) and current_val > 0:
            continue

        name = row["name"]
        norm_key = normalize_name(name)

        if norm_key in kworb_cache:
            listeners, daily = kworb_cache[norm_key]
            df.at[idx, "spotify_monthly_listeners"] = listeners
            if daily is not None:
                df.at[idx, "spotify_daily_change"] = daily
            df.at[idx, "spotify_updated_at"] = now_utc
            df.at[idx, "spotify_data_source"] = "kworb_listeners_deep_sweep"
            matched += 1
            print(f"  [MATCHED] {name} -> Listeners: {listeners:,} | Daily Change: {daily}")

    df.to_csv(DATA_PATH, index=False)
    final_valid = df["spotify_monthly_listeners"].notna().sum()

    print("\n==========================================")
    print("Kworb Sweep Completed!")
    print(f"Newly Matched:  {matched}")
    print(f"Total Coverage: {final_valid} / {len(df)} ({(final_valid/len(df))*100:.1f}%)")
    print(f"Saved directly to: {DATA_PATH}")
    print("==========================================")


if __name__ == "__main__":
    run_sweep()