hiffi-artist-intelligence/scripts/verify_artists_spotify_registry.py

import os
import re
import time
import base64
import pandas as pd
from curl_cffi import requests
from dotenv import load_dotenv

# Load variables directly from .env file
load_dotenv()

DATA_DIR = "data"
INPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
OUTPUT_PATH = os.path.join(DATA_DIR, "master_artists_intelligence_17k_final.csv")
CHECKPOINT_PATH = os.path.join(DATA_DIR, "checkpoints", "spotify_verification_checkpoint.csv")

os.makedirs(os.path.join(DATA_DIR, "checkpoints"), exist_ok=True)

SPOTIFY_CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
SPOTIFY_CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")


def get_spotify_token(client_id: str, client_secret: str) -> str:
    auth_str = f"{client_id}:{client_secret}"
    b64_auth = base64.b64encode(auth_str.encode()).decode()
    resp = requests.post(
        "https://accounts.spotify.com/api/token",
        data={"grant_type": "client_credentials"},
        headers={"Authorization": f"Basic {b64_auth}"},
        timeout=15
    )
    if resp.status_code != 200:
        raise ValueError(f"Spotify authentication failed (HTTP {resp.status_code}): {resp.text}")
    return resp.json()["access_token"]


def normalize_name(text: str) -> str:
    if not text:
        return ""
    cleaned = re.sub(r"\(.*?\)|\[.*?\]", "", str(text))
    cleaned = re.sub(r"[^\w\s]", "", cleaned)
    return cleaned.strip().lower()


def run_spotify_verification():
    if not SPOTIFY_CLIENT_ID or not SPOTIFY_CLIENT_SECRET:
        print("ERROR: SPOTIFY_CLIENT_ID or SPOTIFY_CLIENT_SECRET not found in .env.")
        return

    print("Authenticating with Spotify using .env credentials...")
    token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
    print("Authentication successful!")

    headers = {"Authorization": f"Bearer {token}"}
    session = requests.Session()

    if not os.path.exists(INPUT_PATH):
        print(f"ERROR: Target file not found at {INPUT_PATH}")
        return

    df = pd.read_csv(INPUT_PATH)

    # Ensure required columns exist
    for col in ["spotify_artist_id", "spotify_verified_followers", "spotify_profile_url"]:
        if col not in df.columns:
            df[col] = None

    # Load existing progress from checkpoint
    processed_artists = set()
    if os.path.exists(CHECKPOINT_PATH):
        chk = pd.read_csv(CHECKPOINT_PATH)
        processed_artists = set(chk["artist_name"].dropna().apply(normalize_name))
        print(f"Resuming: {len(processed_artists):,} artists already logged in checkpoint.")

    # Target unverified tier records
    unverified_mask = df["catalog_intelligence_tier"] == "TIER_3_UNAUDITED_DIGITAL_CATALOG"
    target_indices = [
        idx for idx in df[unverified_mask].index
        if normalize_name(df.at[idx, "artist_name"]) not in processed_artists
    ]

    print(f"Verifying {len(target_indices):,} catalog artists directly on Spotify...\n")

    token_time = time.time()
    checkpoint_records = []
    matched_count = 0

    for i, idx in enumerate(target_indices, start=1):
        a_name = str(df.at[idx, "artist_name"])
        target_norm = normalize_name(a_name)

        # Refresh bearer token every 50 minutes (tokens expire after 60 minutes)
        if time.time() - token_time > 3000:
            token = get_spotify_token(SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET)
            headers = {"Authorization": f"Bearer {token}"}
            token_time = time.time()

        url = f"https://api.spotify.com/v1/search?q={requests.utils.quote(a_name)}&type=artist&limit=5"

        try:
            resp = session.get(url, headers=headers, timeout=10)
            if resp.status_code == 429:
                retry_after = int(resp.headers.get("Retry-After", 5))
                print(f"Rate limited. Pausing for {retry_after}s...")
                time.sleep(retry_after)
                resp = session.get(url, headers=headers, timeout=10)

            if resp.status_code == 200:
                items = resp.json().get("artists", {}).get("items", [])
                matched_item = None

                for item in items:
                    cand_norm = normalize_name(item.get("name", ""))
                    if cand_norm == target_norm:
                        matched_item = item
                        break

                if matched_item:
                    sp_id = matched_item["id"]
                    followers = matched_item.get("followers", {}).get("total", 0)
                    sp_url = matched_item.get("external_urls", {}).get("spotify", "")

                    df.at[idx, "spotify_artist_id"] = sp_id
                    df.at[idx, "spotify_verified_followers"] = followers
                    df.at[idx, "spotify_profile_url"] = sp_url
                    df.at[idx, "catalog_intelligence_tier"] = "TIER_3_SPOTIFY_VERIFIED_ENTITY"
                    matched_count += 1

            checkpoint_records.append({
                "artist_name": a_name,
                "spotify_artist_id": df.at[idx, "spotify_artist_id"],
                "spotify_verified_followers": df.at[idx, "spotify_verified_followers"],
                "spotify_profile_url": df.at[idx, "spotify_profile_url"],
                "catalog_intelligence_tier": df.at[idx, "catalog_intelligence_tier"]
            })

        except Exception as e:
            pass

        # Checkpoint every 100 artists
        if i % 100 == 0:
            chk_df = pd.DataFrame(checkpoint_records)
            if os.path.exists(CHECKPOINT_PATH):
                chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
            else:
                chk_df.to_csv(CHECKPOINT_PATH, index=False)
            checkpoint_records = []
            print(f"[{i:,}/{len(target_indices):,}] Checked | Verified on Spotify: {matched_count:,}")

    # Write remaining checkpoint records
    if checkpoint_records:
        chk_df = pd.DataFrame(checkpoint_records)
        if os.path.exists(CHECKPOINT_PATH):
            chk_df.to_csv(CHECKPOINT_PATH, mode="a", header=False, index=False)
        else:
            chk_df.to_csv(CHECKPOINT_PATH, index=False)

    df.to_csv(OUTPUT_PATH, index=False)
    print(f"\nVerification complete! Final catalog saved to: {OUTPUT_PATH}")


if __name__ == "__main__":
    run_spotify_verification()