feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort

collectors/sync_remaining_spotify.py

@@ -0,0 +1,138 @@
+"""
+collectors/sync_remaining_spotify.py
+Pulls verified monthly listener metrics for the 353 non-charting artists
+via Spotify's public Embed & Partner JSON endpoints.
+"""
+
+import re
+import time
+import sqlite3
+import logging
+from pathlib import Path
+from datetime import datetime, timezone
+import requests
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger("SpotifyTailSync")
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"
+
+HTTP_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9"
+}
+
+def extract_spotify_id(url):
+    m = re.search(r"artist/([0-9A-Za-z]{22})", str(url))
+    return m.group(1) if m else None
+
+def get_spotify_web_token(session):
+    """Fetches an ephemeral anonymous web client bearer token directly from Spotify."""
+    try:
+        url = "https://open.spotify.com/get_access_token?reason=transport&productType=web_player"
+        r = session.get(url, timeout=6)
+        if r.status_code == 200:
+            return r.json().get("accessToken")
+    except Exception:
+        pass
+    return None
+
+def fetch_monthly_listeners(session, sp_id, token=None):
+    # Strategy 1: Public Embed JSON resource
+    embed_url = f"https://open.spotify.com/embed/artist/{sp_id}"
+    try:
+        r = session.get(embed_url, timeout=6)
+        if r.status_code == 200:
+            # Pattern: "monthlyListeners":123456
+            m = re.search(r'\"monthlyListeners\":\s*(\d+)', r.text)
+            if m:
+                return int(m.group(1))
+            
+            # Pattern: "123,456 monthly listeners"
+            m2 = re.search(r'([\d,]+)\s+monthly listeners', r.text, re.IGNORECASE)
+            if m2:
+                return int(m2.group(1).replace(",", ""))
+    except Exception:
+        pass
+
+    # Strategy 2: Anonymous partner GraphQL / metadata endpoint
+    if token:
+        try:
+            api_url = f"https://api-partner.spotify.com/pathfinder/v1/query"
+            params = {
+                "operationName": "queryArtistOverview",
+                "variables": f'{{"uris":["spotify:artist:{sp_id}"]}}',
+                "extensions": '{"persistedQuery":{"version":1,"sha256Hash":"35ac4265af99890eac3ff8492314508c82a647970977821c4fa48956828551a3"}}'
+            }
+            h = dict(HTTP_HEADERS)
+            h["Authorization"] = f"Bearer {token}"
+            res = session.get(api_url, params=params, headers=h, timeout=6)
+            if res.status_code == 200:
+                data = res.json()
+                count = (data.get("data", {})
+                             .get("artists", [{}])[0]
+                             .get("stats", {})
+                             .get("monthlyListeners"))
+                if count:
+                    return int(count)
+        except Exception:
+            pass
+
+    return None
+
+def run():
+    conn = sqlite3.connect(DB_PATH)
+    cur = conn.cursor()
+
+    # Artists who were not on Kworb's top 2,500 chart
+    cur.execute("""
+        SELECT a.artist_id, a.artist_name, a.spotify_url, t.spotify_monthly_listeners
+        FROM artists a
+        JOIN telemetry_metrics t ON a.artist_id = t.artist_id
+        WHERE a.image_verified_flag = 1 
+          AND (t.spotify_daily_change = 0 OR t.spotify_daily_change IS NULL)
+    """)
+    unresolved = cur.fetchall()
+    logger.info(f"Targeting {len(unresolved)} remaining non-charting artists...")
+
+    session = requests.Session()
+    session.headers.update(HTTP_HEADERS)
+
+    token = get_spotify_web_token(session)
+    if token:
+        logger.info("Acquired anonymous client access token for Spotify web service.")
+
+    now_iso = datetime.now(timezone.utc).isoformat()
+    updated = 0
+
+    for idx, (artist_id, name, sp_url, current_val) in enumerate(unresolved, 1):
+        sp_id = extract_spotify_id(sp_url)
+        if not sp_id:
+            continue
+
+        listeners = fetch_monthly_listeners(session, sp_id, token)
+        if listeners and listeners > 0:
+            cur.execute("""
+                UPDATE telemetry_metrics
+                SET spotify_monthly_listeners = ?, last_synced_at = ?
+                WHERE artist_id = ?
+            """, (listeners, now_iso, artist_id))
+            updated += 1
+            logger.info(f"[{idx}/{len(unresolved)}] {name} -> {listeners:,} monthly listeners")
+        else:
+            logger.debug(f"[{idx}/{len(unresolved)}] Could not resolve {name}")
+
+        if idx % 25 == 0:
+            conn.commit()
+            time.sleep(0.2)
+        else:
+            time.sleep(0.08)
+
+    conn.commit()
+    conn.close()
+    logger.info(f"Tail sync finished. Updated {updated} / {len(unresolved)} artists.")
+
+if __name__ == "__main__":
+    run()