feat: finalize empirical ordinal tier rankings, fix entity collisions, and dual-mode dashboard

scripts/verify_artists_spotify_registry.py

@@ -0,0 +1,163 @@
+import os
+import re
+import time
+import base64
+import pandas as pd
+from curl_cffi import requests
+from dotenv import load_dotenv
+
+# Load variables directly from .env file
+load_dotenv()
+
+DATA_DIR = "data"
+INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
+CHECKPOINT_PATH = os.path.join(DATA_DIR, "checkpoints", "spotify_verification_checkpoint.csv")
+
+os.makedirs(os.path.join(DATA_DIR, "checkpoints"), exist_ok=True)
+
+SPOTIFY_CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
+SPOTIFY_CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")
+
+
+def get_spotify_token(client_id: str, client_secret: str) -> str:
+    auth_str = f"{client_id}:{client_secret}"
+    b64_auth = base64.b64encode(auth_str.encode()).decode()
+    resp = requests.post(
+        "https://accounts.spotify.com/api/token",
+        data={"grant_type": "client_credentials"},
+        headers={"Authorization": f"Basic {b64_auth}"},
+        timeout=15
+    )
+    if resp.status_code != 200:
+        raise ValueError(f"Spotify authentication failed (HTTP {resp.status_code}): {resp.text}")
+    return resp.json()["access_token"]
+
+
+def normalize_name(text: str) -> str:
+    if not text:
+        return ""
+    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
+    cleaned = re.sub(r"[^\w\s]", "", cleaned)
+    return cleaned.strip().lower()
+
+
+def run_spotify_verification():
+    if not SPOTIFY_CLIENT_ID or not SPOTIFY_CLIENT_SECRET:
+        print("ERROR: SPOTIFY_CLIENT_ID or SPOTIFY_CLIENT_SECRET not found in .env.")
+        return
+
+    print("Authenticating with Spotify using .env credentials...")
+    token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
+    print("Authentication successful!")
+
+    headers = {"Authorization": f"Bearer {token}"}
+    session = requests.Session()
+
+    if not os.path.exists(INPUT_PATH):
+        print(f"ERROR: Target file not found at {INPUT_PATH}")
+        return
+
+    df = pd.read_csv(INPUT_PATH)
+
+    # Ensure required columns exist
+    for col in ["spotify_artist_id", "spotify_verified_followers", "spotify_profile_url"]:
+        if col not in df.columns:
+            df[col] = None
+
+    # Load existing progress from checkpoint
+    processed_artists = set()
+    if os.path.exists(CHECKPOINT_PATH):
+        chk = pd.read_csv(CHECKPOINT_PATH)
+        processed_artists = set(chk["artist_name"].dropna().apply(normalize_name))
+        print(f"Resuming: {len(processed_artists):,} artists already logged in checkpoint.")
+
+    # Target unverified tier records
+    unverified_mask = df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG"
+    target_indices = [
+        idx for idx in df[unverified_mask].index
+        if normalize_name(df.at[idx, "artist_name"]) not in processed_artists
+    ]
+
+    print(f"Verifying {len(target_indices):,} catalog artists directly on Spotify...\n")
+
+    token_time = time.time()
+    checkpoint_records = []
+    matched_count = 0
+
+    for i, idx in enumerate(target_indices, start=1):
+        a_name = str(df.at[idx, "artist_name"])
+        target_norm = normalize_name(a_name)
+
+        # Refresh bearer token every 50 minutes (tokens expire after 60 minutes)
+        if time.time() - token_time > 3000:
+            token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
+            headers = {"Authorization": f"Bearer {token}"}
+            token_time = time.time()
+
+        url = f"https://api.spotify.com/v1/search?q={requests.utils.quote(a_name)}&type=artist&limit=5"
+
+        try:
+            resp = session.get(url, headers=headers, timeout=10)
+            if resp.status_code == 429:
+                retry_after = int(resp.headers.get("Retry-After", 5))
+                print(f"Rate limited. Pausing for {retry_after}s...")
+                time.sleep(retry_after)
+                resp = session.get(url, headers=headers, timeout=10)
+
+            if resp.status_code == 200:
+                items = resp.json().get("artists", {}).get("items", [])
+                matched_item = None
+
+                for item in items:
+                    cand_norm = normalize_name(item.get("name", ""))
+                    if cand_norm == target_norm:
+                        matched_item = item
+                        break
+
+                if matched_item:
+                    sp_id = matched_item["id"]
+                    followers = matched_item.get("followers", {}).get("total", 0)
+                    sp_url = matched_item.get("external_urls", {}).get("spotify", "")
+
+                    df.at[idx, "spotify_artist_id"] = sp_id
+                    df.at[idx, "spotify_verified_followers"] = followers
+                    df.at[idx, "spotify_profile_url"] = sp_url
+                    df.at[idx, "catalog_intelligence_tier"] = "TIER_3_SPOTIFY_VERIFIED_ENTITY"
+                    matched_count += 1
+
+            checkpoint_records.append({
+                "artist_name": a_name,
+                "spotify_artist_id": df.at[idx, "spotify_artist_id"],
+                "spotify_verified_followers": df.at[idx, "spotify_verified_followers"],
+                "spotify_profile_url": df.at[idx, "spotify_profile_url"],
+                "catalog_intelligence_tier": df.at[idx, "catalog_intelligence_tier"]
+            })
+
+        except Exception as e:
+            pass
+
+        # Checkpoint every 100 artists
+        if i % 100 == 0:
+            chk_df = pd.DataFrame(checkpoint_records)
+            if os.path.exists(CHECKPOINT_PATH):
+                chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
+            else:
+                chk_df.to_csv(CHECKPOINT_PATH, index=False)
+            checkpoint_records = []
+            print(f"[{i:,}/{len(target_indices):,}] Checked | Verified on Spotify: {matched_count:,}")
+
+    # Write remaining checkpoint records
+    if checkpoint_records:
+        chk_df = pd.DataFrame(checkpoint_records)
+        if os.path.exists(CHECKPOINT_PATH):
+            chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
+        else:
+            chk_df.to_csv(CHECKPOINT_PATH, index=False)
+
+    df.to_csv(OUTPUT_PATH, index=False)
+    print(f"\nVerification complete! Final catalog saved to: {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    run_spotify_verification()