feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort

scripts/scrape_spotify_218.py

@@ -0,0 +1,192 @@
+"""
+scripts/scrape_spotify_218.py
+Extracts verified Spotify followers and monthly listeners for the cohort artists
+and writes ground truth directly into SQLite.
+"""
+
+import base64
+import json
+import os
+import re
+import sys
+import time
+from pathlib import Path
+import pandas as pd
+import requests
+from dotenv import load_dotenv
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+load_dotenv(ROOT_DIR / ".env")
+
+from core.database import get_connection
+
+CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
+CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")
+
+HEADERS_WEB = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+
+def get_spotify_token():
+    """Fetches an official application access token from Spotify Web API."""
+    if not CLIENT_ID or not CLIENT_SECRET:
+        return None
+    auth_str = f"{CLIENT_ID}:{CLIENT_SECRET}"
+    b64_auth = base64.b64encode(auth_str.encode()).decode()
+    url = "https://accounts.spotify.com/api/token"
+    headers = {"Authorization": f"Basic {b64_auth}"}
+    data = {"grant_type": "client_credentials"}
+    try:
+        r = requests.post(url, headers=headers, data=data, timeout=10)
+        if r.status_code == 200:
+            return r.json().get("access_token")
+    except Exception:
+        pass
+    return None
+
+
+def search_spotify_artist(name, token):
+    """Searches official Spotify API for canonical artist ID and follower count."""
+    if not token:
+        return None
+    url = "https://api.spotify.com/v1/search"
+    headers = {"Authorization": f"Bearer {token}"}
+    params = {"q": name, "type": "artist", "limit": 1}
+    try:
+        r = requests.get(url, headers=headers, params=params, timeout=10)
+        if r.status_code == 200:
+            items = r.json().get("artists", {}).get("items", [])
+            if items:
+                artist = items[0]
+                return {
+                    "id": artist["id"],
+                    "name": artist["name"],
+                    "followers": artist.get("followers", {}).get("total", 0),
+                    "popularity": artist.get("popularity", 0),
+                }
+    except Exception:
+        pass
+    return None
+
+
+def scrape_monthly_listeners(spotify_id):
+    """Parses monthly listeners from Spotify public artist profile page."""
+    url = f"https://open.spotify.com/artist/{spotify_id}"
+    try:
+        r = requests.get(url, headers=HEADERS_WEB, timeout=10)
+        if r.status_code == 200:
+            # Match monthlyListeners in JSON hydration or state payload
+            match_json = re.search(r'"monthlyListeners":\s*(\d+)', r.text)
+            if match_json:
+                return int(match_json.group(1))
+
+            match_text = re.search(r'([\d,]+)\s+monthly\s+listeners', r.text, re.IGNORECASE)
+            if match_text:
+                return int(match_text.group(1).replace(",", ""))
+    except Exception:
+        pass
+    return 0
+
+
+def main():
+    print("=" * 65)
+    print("HIFFI INTELLIGENCE: SPOTIFY 218 LIVE EXTRACTION PIPELINE")
+    print("=" * 65)
+
+    token = get_spotify_token()
+    if not token:
+        print("[!] ERROR: SPOTIFY_CLIENT_ID or SPOTIFY_CLIENT_SECRET is missing or invalid in .env")
+        return
+
+    # Check for cohort CSV
+    csv_path = ROOT_DIR / "data" / "hiffi_multiplatform_500.csv"
+    if not csv_path.exists():
+        csv_path = ROOT_DIR / "cleaned_artists_inventory.csv"
+
+    df = pd.read_csv(csv_path)
+    name_col = "name" if "name" in df.columns else "artist_name"
+    cohort_names = [str(n).strip() for n in df[name_col].dropna()]
+
+    conn = get_connection()
+    cur = conn.cursor()
+
+    # Identify artists missing verified monthly listeners in DB
+    cur.execute("""
+        SELECT a.artist_name 
+        FROM artists a
+        JOIN telemetry_metrics t ON a.artist_id = t.artist_id
+        WHERE t.spotify_monthly_listeners IS NULL OR t.spotify_monthly_listeners = 0
+    """)
+    rows = cur.fetchall()
+    missing_targets = [r[0] for r in rows if r[0] in cohort_names]
+
+    if not missing_targets:
+        print("[*] All artists already have verified monthly listeners. Checking cohort followers...")
+        missing_targets = cohort_names[:50]
+
+    print(f"Targeting {len(missing_targets)} artists for Spotify collection...\n")
+
+    checkpoint_records = []
+    success = 0
+
+    for idx, artist_name in enumerate(missing_targets, 1):
+        artist_meta = search_spotify_artist(artist_name, token)
+        listeners = 0
+        followers = 0
+        sp_id = None
+
+        if artist_meta:
+            sp_id = artist_meta["id"]
+            followers = artist_meta["followers"]
+            listeners = scrape_monthly_listeners(sp_id)
+
+        # Update DB if we resolved listeners or followers
+        if listeners > 0 or followers > 0:
+            success += 1
+            if listeners > 0:
+                cur.execute("""
+                    UPDATE telemetry_metrics
+                    SET spotify_monthly_listeners = ?
+                    WHERE artist_id IN (
+                        SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
+                    )
+                """, (listeners, artist_name))
+
+            print(f"[{idx:>3}/{len(missing_targets)}] {artist_name:<25} -> {listeners:,} monthly listeners | {followers:,} followers")
+        else:
+            print(f"[{idx:>3}/{len(missing_targets)}] {artist_name:<25} -> Unresolved on Spotify")
+
+        checkpoint_records.append({
+            "name": artist_name,
+            "spotify_id": sp_id,
+            "spotify_monthly_listeners": listeners,
+            "spotify_followers": followers,
+        })
+
+        time.sleep(0.3)
+
+    conn.commit()
+    conn.close()
+
+    chk_dir = ROOT_DIR / "data" / "checkpoints"
+    chk_dir.mkdir(parents=True, exist_ok=True)
+    chk_path = chk_dir / "spotify_verification_checkpoint.csv"
+    pd.DataFrame(checkpoint_records).to_csv(chk_path, index=False)
+
+    print("\n" + "=" * 65)
+    print(f"Finished. Extracted metrics for {success} / {len(missing_targets)} artists.")
+    print(f"Checkpoint saved to: {chk_path}")
+    print("=" * 65)
+
+
+if __name__ == "__main__":
+    main()