hiffi-artist-intelligence/scripts/fast_sales_ingestion_17k.py

"""
scripts/fast_sales_ingestion_17k.py
High-speed multithreaded Wikipedia pure sales crawler for HIFFI catalog.
Directly populates pure_album_sales, audited_sales_units, and provenance_flag.
"""

import sqlite3
import json
import time
import re
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
import threading

DB_PATH = "data/hiffi_catalog.db"
WIKI_API_URL = "https://en.wikipedia.org/w/api.php"
HEADERS = {"User-Agent": "HIFFI-FastScraper/1.0 (sri.lekha@superlabs.co)"}
MAX_WORKERS = 8
db_lock = threading.Lock()


def clean_artist_query(name: str) -> str:
    clean = re.sub(r'[,$\'\"]', ' ', name)
    return re.sub(r'\s+', ' ', clean).strip()


def search_best_discography_title(artist_name: str) -> str:
    clean_name = clean_artist_query(artist_name)
    queries = [
        f"{artist_name} albums discography",
        f"{clean_name} discography",
        f"{artist_name} discography",
        artist_name
    ]
    for q in queries:
        params = {
            "action": "opensearch",
            "search": q,
            "limit": "4",
            "namespace": "0",
            "format": "json"
        }
        url = f"{WIKI_API_URL}?{urllib.parse.urlencode(params)}"
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            with urllib.request.urlopen(req, timeout=6) as resp:
                data = json.loads(resp.read().decode("utf-8"))
                titles = data[1] if len(data) > 1 else []
                for t in titles:
                    if "albums discography" in t.lower():
                        return t
                for t in titles:
                    if "discography" in t.lower():
                        return t
                if titles:
                    return titles[0]
        except Exception:
            continue
    return f"{clean_name} discography"


def parse_pure_sales_from_wikitext(wikitext: str) -> float:
    pure_sales = 0.0
    sales_patterns = [
        r'sales\s*=\s*([0-9,]+)',
        r'sales\s*=\s*approx\.\s*([0-9,]+)',
        r'US:\s*([0-9,]+)',
        r'WW:\s*([0-9,]+)',
        r'([0-9,]+)\s*(?:copies|units)\s*sold'
    ]
    for pattern in sales_patterns:
        matches = re.findall(pattern, wikitext, re.IGNORECASE)
        for m in matches:
            cleaned = re.sub(r'[^0-9]', '', m)
            if cleaned and len(cleaned) >= 4:
                val = float(cleaned)
                if val < 500000000:
                    pure_sales += val

    cert_weights = {
        r'\|\s*Gold\b': 500000.0,
        r'\|\s*Platinum\b': 1000000.0,
        r'\|\s*2×\s*Platinum\b': 2000000.0,
        r'\|\s*3×\s*Platinum\b': 3000000.0,
        r'\|\s*Diamond\b': 10000000.0
    }
    for cert_regex, cert_val in cert_weights.items():
        count = len(re.findall(cert_regex, wikitext, re.IGNORECASE))
        if count > 0:
            pure_sales += (count * cert_val)

    return pure_sales


def process_single_artist(artist: dict) -> tuple:
    aid = artist["artist_id"]
    name = artist["artist_name"]

    discog_title = search_best_discography_title(name)
    params = {
        "action": "query",
        "prop": "revisions",
        "rvprop": "content",
        "rvslots": "main",
        "titles": discog_title,
        "format": "json"
    }
    url = f"{WIKI_API_URL}?{urllib.parse.urlencode(params)}"
    sales_units = 0.0

    try:
        req = urllib.request.Request(url, headers=HEADERS)
        with urllib.request.urlopen(req, timeout=8) as resp:
            data = json.loads(resp.read().decode("utf-8"))
            pages = data.get("query", {}).get("pages", {})
            for pid, pdata in pages.items():
                if pid != "-1":
                    revisions = pdata.get("revisions", [])
                    if revisions:
                        wikitext = revisions[0].get("slots", {}).get("main", {}).get("*", "")
                        sales_units = parse_pure_sales_from_wikitext(wikitext)
    except Exception:
        pass

    if sales_units > 0:
        prov = "WIKI_DISCOGRAPHY_AUDIT"
        awards = int(sales_units // 500000)
    else:
        sales_units, awards = 0.0, 0
        prov = "UNVERIFIED_NO_RECORDS"

    return (sales_units, sales_units, awards, prov, aid)


def run_fast_ingestion(limit=3000):
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cur = conn.cursor()

    rows = cur.execute("""
        SELECT a.artist_id, a.artist_name 
        FROM artists a
        JOIN commercial_sales cs ON a.artist_id = cs.artist_id
        WHERE (cs.pure_album_sales IS NULL OR cs.pure_album_sales = 0)
          AND (cs.provenance_flag IS NULL OR cs.provenance_flag NOT IN ('VERIFIED_CSPC', 'VERIFIED_RIAA', 'WIKI_DISCOGRAPHY_AUDIT'))
        ORDER BY a.artist_id ASC
        LIMIT ?
    """, (limit,)).fetchall()

    total = len(rows)
    if total == 0:
        print("✓ All catalog artists already audited with pure sales data.")
        conn.close()
        return

    print(f"🚀 Starting high-speed ingestion for {total} artists with {MAX_WORKERS} concurrent threads...")
    start_time = time.time()

    update_sql = """
        UPDATE commercial_sales
        SET pure_album_sales = ?,
            audited_sales_units = ?,
            awards_count = ?,
            provenance_flag = ?
        WHERE artist_id = ?
    """

    completed = 0
    buffer = []

    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(process_single_artist, dict(r)): r for r in rows}
        for future in as_completed(futures):
            try:
                record = future.result()
                buffer.append(record)
                completed += 1

                if len(buffer) >= 25:
                    with db_lock:
                        cur.executemany(update_sql, buffer)
                        conn.commit()
                    elapsed = time.time() - start_time
                    rate = round(completed / elapsed, 1)
                    print(f"  ⚡ Ingested {completed}/{total} | {rate} artists/sec | Elapsed: {int(elapsed)}s")
                    buffer.clear()

            except Exception:
                pass

        if buffer:
            with db_lock:
                cur.executemany(update_sql, buffer)
                conn.commit()

    conn.close()
    total_time = round(time.time() - start_time, 1)
    print(f"✓ Fast batch complete: {completed} artists processed in {total_time}s.")


if __name__ == "__main__":
    import sys
    count = int(sys.argv[1]) if len(sys.argv) > 1 else 3000
    run_fast_ingestion(count)