feat: finalize empirical ordinal tier rankings, fix entity collisions, and dual-mode dashboard

scripts/apply_catalog_intelligence_tiers.py

@@ -0,0 +1,32 @@
+import os
+import pandas as pd
+
+DATA_DIR = "data"
+SALES_PATH = os.path.join(DATA_DIR, "master_sales_17k_verified.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+
+df = pd.read_csv(SALES_PATH)
+
+
+def assign_tier(row):
+  prov = str(row.get("provenance", ""))
+  units = row.get("audited_sales_units", 0)
+
+  if prov == "VERIFIED_CSPC":
+    return "TIER_1_GLOBAL_SUPERSTAR_CSPC"
+  elif prov == "VERIFIED_RIAA":
+    if pd.notna(units) and units >= 10_000_000:
+      return "TIER_2A_MAJOR_COMMERCIAL_CORE"
+    return "TIER_2B_CERTIFIED_CATALOG"
+
+  # Provenance UNVERIFIED_NO_RECORDS
+  return "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+
+
+df["catalog_intelligence_tier"] = df.apply(assign_tier, axis=1)
+
+print("--- CATALOG INTELLIGENCE TIER DISTRIBUTION ---")
+print(df["catalog_intelligence_tier"].value_counts())
+
+df.to_csv(OUTPUT_PATH, index=False)
+print(f"\nFinal catalog exported to: {OUTPUT_PATH}")

scripts/audit_sales_catalog.py

@@ -0,0 +1,20 @@
+import pandas as pd
+
+FILE_PATH = "data/master_sales_17k_verified.csv"
+df = pd.read_csv(FILE_PATH)
+
+print(f"Total rows in dataset: {len(df):,}")
+print("\n--- PROVENANCE BREAKDOWN ---")
+print(df["provenance"].value_counts())
+
+verified_df = df[df["provenance"] != "UNVERIFIED_NO_RECORDS"]
+print(f"\nTotal Verified Artists: {len(verified_df):,} ({(len(verified_df)/len(df))*100:.2f}%)")
+print(f"Total Unverified Artists: {len(df) - len(verified_df):,}")
+
+top_cols = ["sales_rank", "artist_name", "audited_sales_units", "provenance"]
+
+print("\n--- TOP 20 ARTISTS BY AUDITED SALES ---")
+print(verified_df[top_cols].sort_values("sales_rank").head(20).to_string(index=False))
+
+print("\n--- BOTTOM 5 VERIFIED ARTISTS (THRESHOLD CHECK) ---")
+print(verified_df[top_cols].sort_values("sales_rank", ascending=False).head(5).to_string(index=False))

scripts/build_verified_sales_catalog.py

@@ -0,0 +1,193 @@
+import os
+import re
+import sys
+import time
+import urllib.parse
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+import pandas as pd
+from rapidfuzz import fuzz
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import threading
+
+DATA_DIR = "data"
+CHECKPOINT_DIR = os.path.join(DATA_DIR, "checkpoints")
+os.makedirs(CHECKPOINT_DIR, exist_ok=True)
+
+MASTER_PATH = os.path.join(DATA_DIR, "master_artists_17k.csv")
+CSPC_PATH = os.path.join(DATA_DIR, "chartmasters_audited.csv")
+CHECKPOINT_FILE = os.path.join(CHECKPOINT_DIR, "sales_pipeline_progress.csv")
+FINAL_OUTPUT = os.path.join(DATA_DIR, "master_sales_17k_verified.csv")
+
+checkpoint_lock = threading.Lock()
+
+def normalize_text(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+def extract_units_from_title(title_text: str) -> int:
+    text = (title_text or "").upper().strip()
+    mult_match = re.search(r"(\d+)\s*X", text)
+    multiplier = int(mult_match.group(1)) if mult_match else 1
+    
+    if "DIAMOND" in text:
+        return max(multiplier, 1) * 10_000_000
+    elif "PLATINUM" in text or "MULTI-PLATINUM" in text:
+        return multiplier * 1_000_000
+    elif "GOLD" in text:
+        return 500_000
+    return 0
+
+def is_strict_match(target_name: str, candidate_artist: str, candidate_title: str) -> bool:
+    c_target = normalize_text(target_name)
+    if not c_target or len(c_target) < 2:
+        return False
+        
+    c_cand_artist = normalize_text(candidate_artist)
+    c_cand_title = normalize_text(candidate_title)
+    
+    if c_target == c_cand_artist:
+        return True
+    if re.search(rf"\b{re.escape(c_target)}\b", c_cand_artist):
+        return True
+    if re.search(rf"\b(feat|ft|featuring|with)\s+.*?\b{re.escape(c_target)}\b", candidate_title.lower()):
+        return True
+    if len(c_target) > 5 and fuzz.token_sort_ratio(c_target, c_cand_artist) >= 88:
+        return True
+        
+    return False
+
+def fetch_riaa(artist_name: str) -> dict:
+    target_clean = normalize_text(artist_name)
+    if not target_clean or len(target_clean) < 2:
+        return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+        
+    encoded = urllib.parse.quote_plus(target_clean)
+    url = f"https://www.riaa.com/gold-platinum/?tab_active=default-award&se={encoded}#search_section"
+    
+    try:
+        resp = requests.get(url, impersonate="chrome120", timeout=12)
+        if resp.status_code != 200:
+            return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+            
+        soup = BeautifulSoup(resp.text, "html.parser")
+        table = soup.find("table")
+        if not table:
+            return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+            
+        rows = table.find_all("tr")
+        total_units = 0
+        award_count = 0
+        
+        for r in rows[1:]:
+            cells = r.find_all("td")
+            if len(cells) < 3:
+                continue
+                
+            row_artist = cells[1].get_text(strip=True)
+            row_title = cells[2].get_text(strip=True)
+            
+            if not is_strict_match(artist_name, row_artist, row_title):
+                continue
+                    
+            img = cells[0].find("img")
+            units = extract_units_from_title(img.get("title", "") if img else "")
+            if units > 0:
+                total_units += units
+                award_count += 1
+                
+        if award_count > 0:
+            return {"units": total_units, "awards": award_count, "provenance": "VERIFIED_RIAA"}
+            
+    except Exception:
+        pass
+        
+    return {"units": 0, "awards": 0, "provenance": "UNVERIFIED_NO_RECORDS"}
+
+def process_artist(row, cspc_dict):
+    a_id = row["artist_id"]
+    a_name = str(row["artist_name"])
+    c_name = normalize_text(a_name)
+    
+    # Tier 1: CSPC
+    if c_name in cspc_dict:
+        c_data = cspc_dict[c_name]
+        return {
+            "artist_id": a_id,
+            "artist_name": a_name,
+            "audited_sales_units": float(c_data["total_eas_cspc"]),
+            "awards_count": None,
+            "provenance": "VERIFIED_CSPC"
+        }
+    
+    # Tier 2: RIAA
+    riaa_res = fetch_riaa(a_name)
+    units = riaa_res["units"]
+    return {
+        "artist_id": a_id,
+        "artist_name": a_name,
+        "audited_sales_units": units if units > 0 else None,
+        "awards_count": riaa_res["awards"] if riaa_res["awards"] > 0 else None,
+        "provenance": riaa_res["provenance"]
+    }
+
+def run_pipeline(max_workers=5):
+    print(f"Initializing Multi-Threaded Sales Pipeline ({max_workers} Workers)...")
+    master_df = pd.read_csv(MASTER_PATH)
+    cspc_df = pd.read_csv(CSPC_PATH)
+    cspc_dict = {normalize_text(r["artist_name"]): r for _, r in cspc_df.iterrows()}
+    
+    if os.path.exists(CHECKPOINT_FILE):
+        progress_df = pd.read_csv(CHECKPOINT_FILE)
+        processed_ids = set(progress_df["artist_id"].dropna())
+        records = progress_df.to_dict("records")
+        print(f"Resuming from checkpoint: {len(processed_ids):,} artists already processed.")
+    else:
+        records = []
+        processed_ids = set()
+
+    queue_df = master_df[~master_df["artist_id"].isin(processed_ids)]
+    total_remaining = len(queue_df)
+    total_catalog = len(master_df)
+    
+    print(f"Running on remaining {total_remaining:,} artists out of {total_catalog:,}...\n")
+    
+    processed_count = len(processed_ids)
+    
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_to_artist = {
+            executor.submit(process_artist, row, cspc_dict): row 
+            for _, row in queue_df.iterrows()
+        }
+        
+        for future in as_completed(future_to_artist):
+            res = future.result()
+            records.append(res)
+            processed_count += 1
+            
+            if res["audited_sales_units"] is not None and res["audited_sales_units"] > 0:
+                print(f"[{processed_count:,}/{total_catalog:,}] {res['artist_name']:<22} -> {res['audited_sales_units']:>12,f} ({res['provenance']})")
+            
+            # Save checkpoint every 50 artists
+            if processed_count % 50 == 0:
+                with checkpoint_lock:
+                    pd.DataFrame(records).to_csv(CHECKPOINT_FILE, index=False)
+
+    # Final Rank & Export
+    df_out = pd.DataFrame(records)
+    has_sales = df_out["audited_sales_units"].notna()
+    df_out["sales_rank"] = None
+    df_out.loc[has_sales, "sales_rank"] = (
+        df_out.loc[has_sales, "audited_sales_units"]
+        .rank(ascending=False, method="min")
+        .astype(int)
+    )
+    
+    df_out.to_csv(FINAL_OUTPUT, index=False)
+    print(f"\nPipeline successfully finished! Final catalog saved to: {FINAL_OUTPUT}")
+
+if __name__ == "__main__":
+    run_pipeline(max_workers=5)

scripts/enrich_with_kworb_listeners.py

@@ -0,0 +1,144 @@
+import os
+import re
+import time
+import pandas as pd
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+
+DATA_DIR = "data"
+SALES_PATH = os.path.join(DATA_DIR, "master_sales_17k_verified.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+
+KWORB_STREAMS_URL = "https://kworb.net/spotify/artists.html"
+KWORB_LISTENERS_URL = "https://kworb.net/spotify/listeners.html"
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def fetch_with_retry(session: requests.Session, url: str, max_retries: int = 3, timeout: int = 45):
+    for attempt in range(1, max_retries + 1):
+        try:
+            resp = session.get(url, impersonate="chrome120", timeout=timeout)
+            if resp.status_code == 200:
+                return resp
+            print(f"Warning: HTTP {resp.status_code} on attempt {attempt}/{max_retries}. Retrying...")
+        except Exception as e:
+            print(f"Connection attempt {attempt}/{max_retries} timed out or failed ({e}). Retrying in 3s...")
+            time.sleep(3)
+    raise ConnectionError(f"Failed to fetch {url} after {max_retries} attempts.")
+
+
+def fetch_kworb_streams(session: requests.Session) -> dict:
+    print("1/2: Fetching Kworb all-time streams table...")
+    resp = fetch_with_retry(session, KWORB_STREAMS_URL, max_retries=3, timeout=45)
+
+    soup = BeautifulSoup(resp.text, "html.parser")
+    table = soup.find("table")
+    streams_data = {}
+
+    for row in table.find_all("tr")[1:]:
+        cells = row.find_all("td")
+        if len(cells) < 2:
+            continue
+        artist_anchor = cells[0].find("a")
+        artist_name = artist_anchor.get_text(strip=True) if artist_anchor else cells[0].get_text(strip=True)
+
+        try:
+            stream_str = cells[1].get_text(strip=True).replace(",", "")
+            total_streams = int(float(stream_str) * 1_000_000)
+        except (ValueError, IndexError):
+            total_streams = 0
+
+        norm_key = normalize_name(artist_name)
+        if norm_key and total_streams > 0:
+            streams_data[norm_key] = total_streams
+
+    print(f"Loaded {len(streams_data):,} stream records from Kworb.")
+    return streams_data
+
+
+def fetch_kworb_listeners(session: requests.Session) -> dict:
+    print("2/2: Fetching Kworb monthly listeners table...")
+    resp = fetch_with_retry(session, KWORB_LISTENERS_URL, max_retries=3, timeout=45)
+
+    soup = BeautifulSoup(resp.text, "html.parser")
+    table = soup.find("table")
+    listeners_data = {}
+
+    for row in table.find_all("tr")[1:]:
+        cells = row.find_all("td")
+        if len(cells) < 3:
+            continue
+        artist_anchor = cells[1].find("a")
+        artist_name = artist_anchor.get_text(strip=True) if artist_anchor else cells[1].get_text(strip=True)
+
+        try:
+            listeners_str = cells[2].get_text(strip=True).replace(",", "")
+            monthly_listeners = int(listeners_str)
+        except (ValueError, IndexError):
+            monthly_listeners = 0
+
+        norm_key = normalize_name(artist_name)
+        if norm_key and monthly_listeners > 0:
+            listeners_data[norm_key] = monthly_listeners
+
+    print(f"Loaded {len(listeners_data):,} monthly listener records from Kworb.")
+    return listeners_data
+
+
+def run_strict_intelligence_enrichment():
+    print("Loading base sales dataset...")
+    df = pd.read_csv(SALES_PATH)
+
+    session = requests.Session()
+    streams_dict = fetch_kworb_streams(session)
+    listeners_dict = fetch_kworb_listeners(session)
+
+    print("Mapping metrics to normalized artist keys...")
+    df["norm_key"] = df["artist_name"].apply(normalize_name)
+
+    # Attach verified stream and listener metrics via exact match
+    df["spotify_total_streams"] = df["norm_key"].map(streams_dict)
+    df["spotify_monthly_listeners"] = df["norm_key"].map(listeners_dict)
+
+    def assign_tier(row):
+        prov = str(row.get("provenance", ""))
+        units = row.get("audited_sales_units", 0)
+
+        # Baseline Audits
+        if prov == "VERIFIED_CSPC":
+            return "TIER_1_GLOBAL_SUPERSTAR_CSPC"
+        elif prov == "VERIFIED_RIAA":
+            if pd.notna(units) and units >= 10_000_000:
+                return "TIER_2A_MAJOR_COMMERCIAL_CORE"
+            return "TIER_2B_CERTIFIED_CATALOG"
+
+        # Streaming Velocity (Exact-match verified only)
+        if pd.notna(row["spotify_total_streams"]) and row["spotify_total_streams"] > 0:
+            return "TIER_3A_VERIFIED_DIGITAL_CATALOG"
+
+        if pd.notna(row["spotify_monthly_listeners"]) and row["spotify_monthly_listeners"] > 0:
+            return "TIER_3B_ACTIVE_MONTHLY_STREAMERS"
+
+        return "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+
+    print("Assigning finalized intelligence tiers...")
+    df["catalog_intelligence_tier"] = df.apply(assign_tier, axis=1)
+
+    df.drop(columns=["norm_key"], inplace=True)
+
+    print("\n--- FINAL HARDENED TIER BREAKDOWN ---")
+    print(df["catalog_intelligence_tier"].value_counts())
+
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nSaved sanitized catalog to: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_strict_intelligence_enrichment()

scripts/enrich_with_kworb_streams.py

@@ -0,0 +1,128 @@
+import os
+import re
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+import pandas as pd
+from rapidfuzz import fuzz
+
+DATA_DIR = "data"
+CATALOG_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+
+KWORB_STREAMS_URL = "https://kworb.net/spotify/artists.html"
+KWORB_LISTENERS_URL = "https://kworb.net/spotify/listeners.html"
+
+
+def normalize_name(text: str) -> str:
+  if not text:
+    return ""
+  cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+  cleaned = re.sub(r"[^\w\s]", "", cleaned)
+  return cleaned.strip().lower()
+
+
+def fetch_kworb_master_table() -> dict:
+  print("Fetching Kworb master stream table...")
+  session = requests.Session()
+  resp = session.get(KWORB_STREAMS_URL, impersonate="chrome120", timeout=15)
+
+  if resp.status_code != 200:
+    raise ConnectionError(
+        f"Failed to fetch Kworb streams table (Status: {resp.status_code})"
+    )
+
+  soup = BeautifulSoup(resp.text, "html.parser")
+  table = soup.find("table")
+  if not table:
+    raise ValueError("No table found on Kworb page.")
+
+  kworb_data = {}
+  rows = table.find_all("tr")
+
+  # Table format: Artist | Total Streams (M) | Daily (M) | As Lead (M) | Solo (M) | As Feature (M)
+  for row in rows[1:]:
+    cells = row.find_all("td")
+    if len(cells) < 2:
+      continue
+
+    artist_anchor = cells[0].find("a")
+    artist_name = (
+        artist_anchor.get_text(strip=True)
+        if artist_anchor
+        else cells[0].get_text(strip=True)
+    )
+
+    try:
+      # Kworb streams are formatted in millions (e.g., '143,818.8' means 143.8 billion)
+      stream_str = cells[1].get_text(strip=True).replace(",", "")
+      total_streams = int(float(stream_str) * 1_000_000)
+    except (ValueError, IndexError):
+      total_streams = 0
+
+    norm_key = normalize_name(artist_name)
+    if norm_key and total_streams > 0:
+      kworb_data[norm_key] = {
+          "kworb_artist_name": artist_name,
+          "spotify_total_streams": total_streams,
+      }
+
+  print(f"Loaded {len(kworb_data):,} artists with lifetime stream metrics.")
+  return kworb_data
+
+
+def run_kworb_enrichment():
+  df = pd.read_csv(CATALOG_PATH)
+  kworb_dict = fetch_kworb_master_table()
+
+  if "spotify_total_streams" not in df.columns:
+    df["spotify_total_streams"] = None
+
+  unverified_mask = df["catalog_intelligence_tier"].str.contains(
+      "TIER_3_UNAUDITED", na=False
+  )
+  unverified_indices = df[unverified_mask].index
+
+  print(
+      f"Matching Kworb streaming metrics against {len(unverified_indices):,}"
+      " unverified artists..."
+  )
+
+  matches = 0
+  for idx in unverified_indices:
+    a_name = str(df.at[idx, "artist_name"])
+    c_name = normalize_name(a_name)
+
+    # 1. Exact match
+    if c_name in kworb_dict:
+      entry = kworb_dict[c_name]
+      df.at[idx, "spotify_total_streams"] = entry["spotify_total_streams"]
+      df.at[idx, "catalog_intelligence_tier"] = (
+          "TIER_3A_VERIFIED_DIGITAL_CATALOG"
+      )
+      matches += 1
+      continue
+
+    # 2. Strict token match for names longer than 5 chars
+    if len(c_name) > 5:
+      for k_key, entry in kworb_dict.items():
+        if fuzz.token_sort_ratio(c_name, k_key) >= 92:
+          df.at[idx, "spotify_total_streams"] = entry["spotify_total_streams"]
+          df.at[idx, "catalog_intelligence_tier"] = (
+              "TIER_3A_VERIFIED_DIGITAL_CATALOG"
+          )
+          matches += 1
+          break
+
+  print(
+      f"\nEnrichment complete! Successfully matched {matches:,} previously"
+      " unverified artists."
+  )
+  print("\nUpdated Catalog Intelligence Tiers:")
+  print(df["catalog_intelligence_tier"].value_counts())
+
+  df.to_csv(OUTPUT_PATH, index=False)
+  print(f"Saved updated catalog to: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+  run_kworb_enrichment()

scripts/filter_recovery_collisions.py

@@ -0,0 +1,46 @@
+import difflib
+import urllib.parse
+import pandas as pd
+
+DATA_PATH = "data/master_artists_intelligence_17k_final.csv"
+
+df = pd.read_csv(DATA_PATH)
+
+recovered_mask = df["catalog_intelligence_tier"] == "TIER_3_LASTFM_VERIFIED_ENTITY"
+
+reverted_count = 0
+for idx in df[recovered_mask].index:
+    orig = str(df.at[idx, "artist_name"]).strip()
+    url = str(df.at[idx, "lastfm_url"]).strip()
+
+    if "/music/" not in url:
+        continue
+
+    # Extract slug after /music/
+    slug = url.split("/music/")[-1]
+    # Remove Last.fm internal redirect markers
+    slug = slug.replace("+noredirect/", "").replace("noredirect/", "")
+    # Unquote URL encodings (%20 -> space, %21 -> !, etc.)
+    matched_name = urllib.parse.unquote_plus(slug).strip()
+
+    # Clean characters for comparison
+    clean_orig = "".join(c for c in orig.lower() if c.isalnum() or c.isspace()).strip()
+    clean_match = "".join(c for c in matched_name.lower() if c.isalnum() or c.isspace()).strip()
+
+    # Direct exception for known collisions
+    if clean_orig == "tkkravitz" and "lenny" in clean_match:
+        similarity = 0.0
+    else:
+        similarity = difflib.SequenceMatcher(None, clean_orig, clean_match).ratio()
+
+    # If the name drifted completely, revert to unaudited
+    if similarity < 0.65:
+        reverted_count += 1
+        print(f"Reverting false collision: '{orig}' -> matched '{matched_name}' (Sim: {similarity:.2f})")
+        df.at[idx, "lastfm_listeners"] = None
+        df.at[idx, "lastfm_playcount"] = None
+        df.at[idx, "lastfm_url"] = None
+        df.at[idx, "catalog_intelligence_tier"] = "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+
+print(f"\nFiltered out {reverted_count} true false collisions.")
+df.to_csv(DATA_PATH, index=False)

scripts/finalize_master_catalog.py

@@ -0,0 +1,72 @@
+import os
+import re
+import pandas as pd
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k.csv")
+FINAL_OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def run_final_qa():
+    print(f"Reading dataset: {INPUT_PATH}")
+    df = pd.read_csv(INPUT_PATH)
+    initial_count = len(df)
+    print(f"Initial total artists: {initial_count:,}")
+
+    # 1. Deduplication by normalized name and artist_id
+    df["norm_key"] = df["artist_name"].apply(normalize_name)
+    
+    # Sort so verified records take priority over unverified during deduplication
+    tier_order = {
+        "TIER_1_GLOBAL_SUPERSTAR_CSPC": 1,
+        "TIER_2A_MAJOR_COMMERCIAL_CORE": 2,
+        "TIER_2B_CERTIFIED_CATALOG": 3,
+        "TIER_3A_VERIFIED_DIGITAL_CATALOG": 4,
+        "TIER_3B_ACTIVE_MONTHLY_STREAMERS": 5,
+        "TIER_3_UNAUDITED_DIGITAL_CATALOG": 6,
+    }
+    df["tier_rank"] = df["catalog_intelligence_tier"].map(tier_order).fillna(99)
+    df.sort_values(by=["tier_rank", "audited_sales_units", "spotify_total_streams", "spotify_monthly_listeners"], 
+                   ascending=[True, False, False, False], inplace=True)
+
+    # Drop duplicate normalized names, retaining highest-ranked tier
+    df.drop_duplicates(subset=["norm_key"], keep="first", inplace=True)
+    deduped_count = len(df)
+    print(f"Deduplicated count: {deduped_count:,} (Removed {initial_count - deduped_count:,} duplicate variants)")
+
+    # 2. Add Unified Sorting Metric Columns for Downstream UI
+    def get_primary_metric(row):
+        tier = row["catalog_intelligence_tier"]
+        if tier in ["TIER_1_GLOBAL_SUPERSTAR_CSPC", "TIER_2A_MAJOR_COMMERCIAL_CORE", "TIER_2B_CERTIFIED_CATALOG"]:
+            return pd.Series(["AUDITED_SALES_UNITS", row["audited_sales_units"]])
+        elif tier == "TIER_3A_VERIFIED_DIGITAL_CATALOG":
+            return pd.Series(["LIFETIME_STREAMS", row["spotify_total_streams"]])
+        elif tier == "TIER_3B_ACTIVE_MONTHLY_STREAMERS":
+            return pd.Series(["MONTHLY_LISTENERS", row["spotify_monthly_listeners"]])
+        return pd.Series(["UNAUDITED", None])
+
+    df[["primary_metric_type", "primary_metric_value"]] = df.apply(get_primary_metric, axis=1)
+
+    # Clean temporary helper columns
+    df.drop(columns=["norm_key", "tier_rank"], inplace=True)
+
+    # Sort final export: Tiered priority first, then primary metric descending
+    df.sort_values(by=["catalog_intelligence_tier", "primary_metric_value"], 
+                   ascending=[True, False], inplace=True)
+
+    df.to_csv(FINAL_OUTPUT_PATH, index=False)
+    print(f"\nFinal hardened intelligence catalog exported to: {FINAL_OUTPUT_PATH}")
+    print("\nFinal Tier Breakdown:")
+    print(df["catalog_intelligence_tier"].value_counts())
+
+
+if __name__ == "__main__":
+    run_final_qa()

scripts/rank_master_catalog.py

@@ -0,0 +1,97 @@
+import os
+import pandas as pd
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+
+
+def run_empirical_catalog_ranking():
+    if not os.path.exists(INPUT_PATH):
+        print(f"ERROR: Dataset not found at {INPUT_PATH}")
+        return
+
+    print(f"Loading master verified catalog: {INPUT_PATH}")
+    df = pd.read_csv(INPUT_PATH)
+
+    # 1. Define Strict Registry Tier Hierarchy
+    tier_priority = {
+        "TIER_1_GLOBAL_SUPERSTAR_CSPC": 1,
+        "TIER_2A_MAJOR_COMMERCIAL_CORE": 2,
+        "TIER_2B_CERTIFIED_CATALOG": 3,
+        "TIER_3A_VERIFIED_DIGITAL_CATALOG": 4,
+        "TIER_3B_ACTIVE_MONTHLY_STREAMERS": 5,
+        "TIER_3_LASTFM_VERIFIED_ENTITY": 6,
+        "TIER_3_UNAUDITED_DIGITAL_CATALOG": 7
+    }
+
+    df["tier_order"] = df["catalog_intelligence_tier"].map(tier_priority).fillna(99)
+
+    # 2. Extract Primary Audited Metric per Tier (No conversions, pure integers)
+    def extract_tier_metric(row):
+        order = row["tier_order"]
+        if order in [1, 2, 3]:
+            # CSPC EAS / RIAA Certified Units
+            val = row.get("audited_sales_units")
+            return float(val) if pd.notna(val) else 0.0
+        elif order == 4:
+            # Kworb Spotify Total Discography Streams
+            val = row.get("spotify_total_streams")
+            return float(val) if pd.notna(val) else 0.0
+        elif order == 5:
+            # Kworb Spotify Peak Monthly Listeners
+            val = row.get("spotify_monthly_listeners")
+            return float(val) if pd.notna(val) else 0.0
+        elif order == 6:
+            # Last.fm Global Verified Listeners
+            val = row.get("lastfm_listeners")
+            return float(val) if pd.notna(val) else 0.0
+        return 0.0
+
+    df["primary_empirical_metric"] = df.apply(extract_tier_metric, axis=1)
+
+    # 3. Sort strictly by Tier Order (1 -> 6), then by Primary Metric (Highest -> Lowest)
+    df.sort_values(
+        by=["tier_order", "primary_empirical_metric"],
+        ascending=[True, False],
+        inplace=True
+    )
+    df.reset_index(drop=True, inplace=True)
+
+    # 4. Generate Global Catalog Rank (Only for verified tiers 1 through 6)
+    verified_mask = df["tier_order"] <= 6
+    df["catalog_rank"] = None
+    df.loc[verified_mask, "catalog_rank"] = range(1, int(verified_mask.sum()) + 1)
+
+    # 5. Generate Peer Tier Rank
+    df["tier_rank"] = None
+    for tier_name in df["catalog_intelligence_tier"].unique():
+        if tier_name != "TIER_3_UNAUDITED_DIGITAL_CATALOG":
+            t_mask = df["catalog_intelligence_tier"] == tier_name
+            df.loc[t_mask, "tier_rank"] = range(1, int(t_mask.sum()) + 1)
+
+    # Clean up temporary helper columns
+    df.drop(columns=["tier_order", "primary_empirical_metric"], inplace=True)
+
+    # Save finalized ranked dataset
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nEmpirical ranking complete! Saved to {OUTPUT_PATH}")
+
+    # Display Top 15 Overall
+    preview_cols = [
+        col for col in ["catalog_rank", "tier_rank", "artist_name", "catalog_intelligence_tier", "audited_sales_units", "lastfm_listeners"]
+        if col in df.columns
+    ]
+    print("\n--- TOP 15 CATALOG ARTISTS ---")
+    print(df[preview_cols].head(15).to_string(index=False))
+
+    # Display Top 5 in Tier 3 Last.fm Verified Entities
+    lastfm_sample = df[df["catalog_intelligence_tier"] == "TIER_3_LASTFM_VERIFIED_ENTITY"]
+    if not lastfm_sample.empty:
+        print("\n--- TOP 5 TIER 3 (LAST.FM VERIFIED) ARTISTS ---")
+        lfm_cols = ["catalog_rank", "tier_rank", "artist_name", "lastfm_listeners", "lastfm_playcount"]
+        print(lastfm_sample[lfm_cols].head(5).to_string(index=False))
+
+
+if __name__ == "__main__":
+    run_empirical_catalog_ranking()

scripts/recover_367_unverified.py

@@ -0,0 +1,105 @@
+import os
+import re
+import time
+import ftfy
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATA_PATH = "data/master_artists_intelligence_17k_final.csv"
+OUTPUT_PATH = "data/master_artists_intelligence_17k_final.csv"
+LASTFM_API_KEY = os.getenv("LASTFM_API_KEY")
+
+
+def clean_artist_query(raw_name: str) -> str:
+    """Fix mojibake encoding and clean handle artifacts."""
+    if not raw_name or pd.isna(raw_name):
+        return ""
+    
+    # 1. Fix Mojibake encoding (ãl -> al, Amoré -> Amoré)
+    name = ftfy.fix_text(str(raw_name))
+
+    # 2. Strip leading numbers attached to handles (e.g. 1dessdior -> dess dior)
+    name = re.sub(r"^[0-9]+([a-zA-Z])", r"\1", name)
+
+    # 3. Strip trailing underscores or dots commonly found in IG handles
+    name = re.sub(r"[_\.]+$", "", name)
+    name = name.replace("_", " ").replace(".", " ")
+
+    # 4. Remove collaboration trails like & the Brand New S
+    if " & the " in name:
+        name = name.split(" & the ")[0]
+
+    return name.strip()
+
+
+def query_lastfm_search(session: requests.Session, query: str) -> dict:
+    url = "https://ws.audioscrobbler.com/2.0/"
+    params = {
+        "method": "artist.search",
+        "artist": query,
+        "api_key": LASTFM_API_KEY,
+        "format": "json",
+        "limit": 3
+    }
+    headers = {"User-Agent": "HIFFIRecovery/1.0 (srilekhar86@gmail.com)"}
+
+    try:
+        resp = session.get(url, params=params, headers=headers, impersonate="chrome120", timeout=8)
+        if resp.status_code == 200:
+            data = resp.json()
+            matches = data.get("results", {}).get("artistmatches", {}).get("artist", [])
+            if isinstance(matches, dict):
+                matches = [matches]
+            if matches:
+                # Pick the top result with registered listeners
+                for match in matches:
+                    listeners = int(match.get("listeners", 0))
+                    if listeners > 0:
+                        return {
+                            "canonical_name": match.get("name"),
+                            "listeners": listeners,
+                            "url": match.get("url")
+                        }
+    except Exception:
+        pass
+    return None
+
+
+def run_recovery():
+    print(f"Loading master catalog: {DATA_PATH}")
+    df = pd.read_csv(DATA_PATH)
+
+    unv_mask = df["catalog_rank"].isna() | (df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG")
+    unv_indices = df[unv_mask].index.tolist()
+    print(f"Targeting {len(unv_indices)} unverified candidates for registry recovery...\n")
+
+    session = requests.Session()
+    recovered = 0
+
+    for idx in unv_indices:
+        raw_name = df.at[idx, "artist_name"]
+        cleaned_query = clean_artist_query(raw_name)
+
+        if not cleaned_query or len(cleaned_query) <= 1:
+            continue
+
+        res = query_lastfm_search(session, cleaned_query)
+        time.sleep(0.2)  # Respect Last.fm rate limits
+
+        if res:
+            df.at[idx, "lastfm_listeners"] = res["listeners"]
+            df.at[idx, "lastfm_url"] = res["url"]
+            df.at[idx, "catalog_intelligence_tier"] = "TIER_3_LASTFM_VERIFIED_ENTITY"
+            recovered += 1
+            print(f"[{recovered}] Recovered: '{raw_name}' -> '{res['canonical_name']}' ({res['listeners']:,} listeners)")
+
+    print(f"\nRecovery pass complete: {recovered} / {len(unv_indices)} artists recovered.")
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"Master file updated: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_recovery()

scripts/recover_unverified_artists.py

@@ -0,0 +1,127 @@
+import os
+import re
+import time
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATA_PATH = "data/master_artists_intelligence_17k_final.csv"
+OUTPUT_PATH = "data/master_artists_intelligence_17k_final.csv"
+LASTFM_API_KEY = os.getenv("LASTFM_API_KEY")
+
+
+def clean_primary_artist(raw_name: str) -> str:
+    """Extracts primary artist if string is a collaboration or feature."""
+    if not raw_name or pd.isna(raw_name):
+        return ""
+    text = str(raw_name)
+    # Strip (feat. ...), [prod. ...], etc.
+    text = re.sub(r"\(.*?\)|\[.*?\]", "", text)
+    # Split on explicit collaboration delimiters and take primary artist
+    for delim in [" feat. ", " ft. ", " featuring ", " Feat. ", " Ft. ", " & ", " x ", " X ", " vs. ", " vs "]:
+        if delim in text:
+            text = text.split(delim)[0]
+    return text.strip()
+
+
+def query_lastfm_search(session: requests.Session, artist_name: str) -> dict:
+    url = "https://ws.audioscrobbler.com/2.0/"
+    params = {
+        "method": "artist.search",
+        "artist": artist_name,
+        "api_key": LASTFM_API_KEY,
+        "format": "json",
+        "limit": 1
+    }
+    headers = {"User-Agent": "HIFFIRecovery/1.0 (annotation783@gmail.com)"}
+
+    try:
+        resp = session.get(url, params=params, headers=headers, impersonate="chrome120", timeout=10)
+        if resp.status_code == 200:
+            data = resp.json()
+            matches = data.get("results", {}).get("artistmatches", {}).get("artist", [])
+            if matches:
+                top_match = matches[0] if isinstance(matches, list) else matches
+                listeners = int(top_match.get("listeners", 0))
+                if listeners > 0:
+                    return {
+                        "verified_name": top_match.get("name"),
+                        "lastfm_listeners": listeners,
+                        "lastfm_url": top_match.get("url"),
+                        "source": "LASTFM_SEARCH"
+                    }
+    except Exception:
+        pass
+    return None
+
+
+def query_musicbrainz(session: requests.Session, artist_name: str) -> dict:
+    """Open authoritative music registry fallback."""
+    url = "https://musicbrainz.org/ws/2/artist/"
+    params = {"query": f'artist:"{artist_name}"', "fmt": "json", "limit": 1}
+    headers = {"User-Agent": "HIFFIArtistIntelligence/1.0 ( srilekhar86@gmail.com )"}
+
+    try:
+        resp = session.get(url, params=params, headers=headers, timeout=10)
+        if resp.status_code == 200:
+            data = resp.json()
+            artists = data.get("artists", [])
+            if artists:
+                top_artist = artists[0]
+                score = int(top_artist.get("score", 0))
+                if score >= 90:  # High confidence registry match
+                    mbid = top_artist.get("id")
+                    return {
+                        "verified_name": top_artist.get("name"),
+                        "lastfm_listeners": None,
+                        "lastfm_url": f"https://musicbrainz.org/artist/{mbid}",
+                        "source": "MUSICBRAINZ_REGISTRY"
+                    }
+    except Exception:
+        pass
+    return None
+
+
+def run_recovery():
+    df = pd.read_csv(DATA_PATH)
+    unverified_mask = df["catalog_rank"].isna() | (df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG")
+    indices_to_recover = df[unverified_mask].index.tolist()
+
+    print(f"Attempting secondary recovery on {len(indices_to_recover):,} unverified artists...\n")
+
+    session = requests.Session()
+    recovered_count = 0
+
+    for idx in indices_to_recover:
+        raw_name = str(df.at[idx, "artist_name"])
+        cleaned = clean_primary_artist(raw_name)
+
+        # 1. Try Last.fm search endpoint with cleaned name
+        res = query_lastfm_search(session, cleaned if cleaned else raw_name)
+        time.sleep(0.3)
+
+        # 2. If no result, try MusicBrainz open registry
+        if not res:
+            res = query_musicbrainz(session, cleaned if cleaned else raw_name)
+            time.sleep(1.0)  # MusicBrainz rate limit compliance
+
+        if res:
+            recovered_count += 1
+            if res["lastfm_listeners"]:
+                df.at[idx, "lastfm_listeners"] = res["lastfm_listeners"]
+            if res["lastfm_url"]:
+                df.at[idx, "lastfm_url"] = res["lastfm_url"]
+
+            tier_label = "TIER_3_LASTFM_VERIFIED_ENTITY" if res["source"] == "LASTFM_SEARCH" else "TIER_3_MUSICBRAINZ_VERIFIED"
+            df.at[idx, "catalog_intelligence_tier"] = tier_label
+            print(f"✅ Recovered: '{raw_name}' -> '{res['verified_name']}' ({tier_label})")
+
+    print(f"\nRecovery complete: Successfully recovered {recovered_count:,} additional entities.")
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"Updated catalog saved to {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_recovery()

scripts/verify_artists_lastfm.py

@@ -0,0 +1,180 @@
+import os
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+CHECKPOINT_PATH = os.path.join(DATA_DIR, "checkpoints", "lastfm_verification_checkpoint.csv")
+
+os.makedirs(os.path.join(DATA_DIR, "checkpoints"), exist_ok=True)
+
+LASTFM_API_KEY = os.getenv("LASTFM_API_KEY")
+NUM_WORKERS = 2  # Safe concurrency limit
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def fetch_artist_lastfm(session: requests.Session, a_name: str) -> dict:
+    target_norm = normalize_name(a_name)
+    url = "https://ws.audioscrobbler.com/2.0/"
+    params = {
+        "method": "artist.getinfo",
+        "artist": a_name,
+        "api_key": LASTFM_API_KEY,
+        "format": "json",
+        "autocorrect": "0"
+    }
+    headers = {"User-Agent": "HIFFIArtistAudit/1.0 (annotation783@gmail.com)"}
+
+    for attempt in range(4):
+        try:
+            resp = session.get(url, params=params, headers=headers, impersonate="chrome120", timeout=12)
+            if resp.status_code == 200:
+                data = resp.json()
+
+                # Handle Last.fm Error 29: Rate Limit Exceeded
+                if data.get("error") == 29:
+                    time.sleep(15)
+                    continue
+
+                artist_data = data.get("artist")
+                if artist_data and "name" in artist_data:
+                    ret_norm = normalize_name(artist_data.get("name", ""))
+                    if ret_norm == target_norm:
+                        stats = artist_data.get("stats", {})
+                        listeners = int(stats.get("listeners", 0))
+                        playcount = int(stats.get("playcount", 0))
+                        url_link = artist_data.get("url", "")
+
+                        return {
+                            "artist_name": a_name,
+                            "lastfm_listeners": listeners,
+                            "lastfm_playcount": playcount,
+                            "lastfm_url": url_link,
+                            "verified": listeners > 0
+                        }
+                break
+
+            elif resp.status_code in [429, 503]:
+                time.sleep(12)
+                continue
+
+        except Exception:
+            time.sleep(3)
+
+    return {
+        "artist_name": a_name,
+        "lastfm_listeners": None,
+        "lastfm_playcount": None,
+        "lastfm_url": None,
+        "verified": False
+    }
+
+
+def run_lastfm_verification():
+    if not LASTFM_API_KEY:
+        print("ERROR: LASTFM_API_KEY not found in .env.")
+        return
+
+    df = pd.read_csv(INPUT_PATH)
+
+    for col in ["lastfm_listeners", "lastfm_playcount", "lastfm_url"]:
+        if col not in df.columns:
+            df[col] = None
+
+    # Load verified artists from checkpoint
+    verified_map = {}
+    if os.path.exists(CHECKPOINT_PATH):
+        chk = pd.read_csv(CHECKPOINT_PATH)
+        for _, row in chk.iterrows():
+            norm_k = normalize_name(row["artist_name"])
+            verified_map[norm_k] = row.to_dict()
+        print(f"Resuming with {len(verified_map):,} verified artists already preserved.")
+
+    # Populate preserved checkpoint rows into the dataframe
+    for idx in df.index:
+        norm_k = normalize_name(df.at[idx, "artist_name"])
+        if norm_k in verified_map:
+            record = verified_map[norm_k]
+            df.at[idx, "lastfm_listeners"] = record.get("lastfm_listeners")
+            df.at[idx, "lastfm_playcount"] = record.get("lastfm_playcount")
+            df.at[idx, "lastfm_url"] = record.get("lastfm_url")
+            df.at[idx, "catalog_intelligence_tier"] = "TIER_3_LASTFM_VERIFIED_ENTITY"
+
+    # Only process rows that have not yet been verified
+    unverified_mask = df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+    target_rows = [
+        (idx, str(df.at[idx, "artist_name"]))
+        for idx in df[unverified_mask].index
+    ]
+
+    total_to_process = len(target_rows)
+    print(f"Verifying {total_to_process:,} unverified artists with rate-safe spacing...\n")
+
+    if total_to_process == 0:
+        print("All artists already verified.")
+        return
+
+    session = requests.Session()
+    checkpoint_batch = []
+    completed_count = 0
+    new_verified = 0
+
+    with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:
+        future_to_idx = {
+            executor.submit(fetch_artist_lastfm, session, name): (idx, name)
+            for idx, name in target_rows
+        }
+
+        for future in as_completed(future_to_idx):
+            idx, name = future_to_idx[future]
+            completed_count += 1
+            time.sleep(0.35)
+
+            try:
+                result = future.result()
+            except Exception:
+                result = {"artist_name": name, "verified": False}
+
+            if result.get("verified"):
+                new_verified += 1
+                df.at[idx, "lastfm_listeners"] = result["lastfm_listeners"]
+                df.at[idx, "lastfm_playcount"] = result["lastfm_playcount"]
+                df.at[idx, "lastfm_url"] = result["lastfm_url"]
+                df.at[idx, "catalog_intelligence_tier"] = "TIER_3_LASTFM_VERIFIED_ENTITY"
+
+                checkpoint_batch.append({
+                    "artist_name": name,
+                    "lastfm_listeners": result["lastfm_listeners"],
+                    "lastfm_playcount": result["lastfm_playcount"],
+                    "lastfm_url": result["lastfm_url"],
+                    "catalog_intelligence_tier": "TIER_3_LASTFM_VERIFIED_ENTITY"
+                })
+
+            # Checkpoint every 50 newly verified artists
+            if len(checkpoint_batch) >= 50 or completed_count == total_to_process:
+                if checkpoint_batch:
+                    chk_df = pd.DataFrame(checkpoint_batch)
+                    chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=not os.path.exists(CHECKPOINT_PATH), index=False)
+                    checkpoint_batch = []
+                print(f"[{completed_count:,}/{total_to_process:,}] Checked | New Verified: {new_verified:,} (Total Verified: {len(verified_map) + new_verified:,})")
+
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nVerification complete! Saved to {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_lastfm_verification()

scripts/verify_artists_spotify_registry.py

@@ -0,0 +1,163 @@
+import os
+import re
+import time
+import base64
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+# Load variables directly from .env file
+load_dotenv()
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+CHECKPOINT_PATH = os.path.join(DATA_DIR, "checkpoints", "spotify_verification_checkpoint.csv")
+
+os.makedirs(os.path.join(DATA_DIR, "checkpoints"), exist_ok=True)
+
+SPOTIFY_CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
+SPOTIFY_CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")
+
+
+def get_spotify_token(client_id: str, client_secret: str) -> str:
+    auth_str = f"{client_id}:{client_secret}"
+    b64_auth = base64.b64encode(auth_str.encode()).decode()
+    resp = requests.post(
+        "https://accounts.spotify.com/api/token",
+        data={"grant_type": "client_credentials"},
+        headers={"Authorization": f"Basic {b64_auth}"},
+        timeout=15
+    )
+    if resp.status_code != 200:
+        raise ValueError(f"Spotify authentication failed (HTTP {resp.status_code}): {resp.text}")
+    return resp.json()["access_token"]
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def run_spotify_verification():
+    if not SPOTIFY_CLIENT_ID or not SPOTIFY_CLIENT_SECRET:
+        print("ERROR: SPOTIFY_CLIENT_ID or SPOTIFY_CLIENT_SECRET not found in .env.")
+        return
+
+    print("Authenticating with Spotify using .env credentials...")
+    token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
+    print("Authentication successful!")
+
+    headers = {"Authorization": f"Bearer {token}"}
+    session = requests.Session()
+
+    if not os.path.exists(INPUT_PATH):
+        print(f"ERROR: Target file not found at {INPUT_PATH}")
+        return
+
+    df = pd.read_csv(INPUT_PATH)
+
+    # Ensure required columns exist
+    for col in ["spotify_artist_id", "spotify_verified_followers", "spotify_profile_url"]:
+        if col not in df.columns:
+            df[col] = None
+
+    # Load existing progress from checkpoint
+    processed_artists = set()
+    if os.path.exists(CHECKPOINT_PATH):
+        chk = pd.read_csv(CHECKPOINT_PATH)
+        processed_artists = set(chk["artist_name"].dropna().apply(normalize_name))
+        print(f"Resuming: {len(processed_artists):,} artists already logged in checkpoint.")
+
+    # Target unverified tier records
+    unverified_mask = df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+    target_indices = [
+        idx for idx in df[unverified_mask].index
+        if normalize_name(df.at[idx, "artist_name"]) not in processed_artists
+    ]
+
+    print(f"Verifying {len(target_indices):,} catalog artists directly on Spotify...\n")
+
+    token_time = time.time()
+    checkpoint_records = []
+    matched_count = 0
+
+    for i, idx in enumerate(target_indices, start=1):
+        a_name = str(df.at[idx, "artist_name"])
+        target_norm = normalize_name(a_name)
+
+        # Refresh bearer token every 50 minutes (tokens expire after 60 minutes)
+        if time.time() - token_time > 3000:
+            token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
+            headers = {"Authorization": f"Bearer {token}"}
+            token_time = time.time()
+
+        url = f"https://api.spotify.com/v1/search?q={requests.utils.quote(a_name)}&type=artist&limit=5"
+
+        try:
+            resp = session.get(url, headers=headers, timeout=10)
+            if resp.status_code == 429:
+                retry_after = int(resp.headers.get("Retry-After", 5))
+                print(f"Rate limited. Pausing for {retry_after}s...")
+                time.sleep(retry_after)
+                resp = session.get(url, headers=headers, timeout=10)
+
+            if resp.status_code == 200:
+                items = resp.json().get("artists", {}).get("items", [])
+                matched_item = None
+
+                for item in items:
+                    cand_norm = normalize_name(item.get("name", ""))
+                    if cand_norm == target_norm:
+                        matched_item = item
+                        break
+
+                if matched_item:
+                    sp_id = matched_item["id"]
+                    followers = matched_item.get("followers", {}).get("total", 0)
+                    sp_url = matched_item.get("external_urls", {}).get("spotify", "")
+
+                    df.at[idx, "spotify_artist_id"] = sp_id
+                    df.at[idx, "spotify_verified_followers"] = followers
+                    df.at[idx, "spotify_profile_url"] = sp_url
+                    df.at[idx, "catalog_intelligence_tier"] = "TIER_3_SPOTIFY_VERIFIED_ENTITY"
+                    matched_count += 1
+
+            checkpoint_records.append({
+                "artist_name": a_name,
+                "spotify_artist_id": df.at[idx, "spotify_artist_id"],
+                "spotify_verified_followers": df.at[idx, "spotify_verified_followers"],
+                "spotify_profile_url": df.at[idx, "spotify_profile_url"],
+                "catalog_intelligence_tier": df.at[idx, "catalog_intelligence_tier"]
+            })
+
+        except Exception as e:
+            pass
+
+        # Checkpoint every 100 artists
+        if i % 100 == 0:
+            chk_df = pd.DataFrame(checkpoint_records)
+            if os.path.exists(CHECKPOINT_PATH):
+                chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
+            else:
+                chk_df.to_csv(CHECKPOINT_PATH, index=False)
+            checkpoint_records = []
+            print(f"[{i:,}/{len(target_indices):,}] Checked | Verified on Spotify: {matched_count:,}")
+
+    # Write remaining checkpoint records
+    if checkpoint_records:
+        chk_df = pd.DataFrame(checkpoint_records)
+        if os.path.exists(CHECKPOINT_PATH):
+            chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
+        else:
+            chk_df.to_csv(CHECKPOINT_PATH, index=False)
+
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nVerification complete! Final catalog saved to: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_spotify_verification()