feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort

collectors/sync_complete_catalog.py

@@ -0,0 +1,259 @@
+"""
+collectors/sync_complete_catalog.py
+Master Production Telemetry Synchronizer for HIFFI Catalog.
+Reads credentials automatically from .env.
+- YouTube Data API v3 (Batch 50 IDs per call)
+- Spotify Web API Client Credentials (Batch 50 IDs per call)
+- Kworb Global Index for Spotify Monthly Listeners
+"""
+
+import os
+import re
+import time
+import base64
+import sqlite3
+import logging
+from pathlib import Path
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Tuple
+import requests
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+
+# Load .env from project root
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+load_dotenv(PROJECT_ROOT / ".env")
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s"
+)
+logger = logging.getLogger("TelemetryMaster")
+
+DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"
+
+HTTP_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+}
+
+
+class TelemetryMasterSync:
+    def __init__(self, db_path: Path = DB_PATH):
+        self.db_path = db_path
+        self.session = requests.Session()
+        self.session.headers.update(HTTP_HEADERS)
+        
+        self.yt_api_key = os.getenv("YOUTUBE_API_KEY", "").strip()
+        self.sp_client_id = os.getenv("SPOTIFY_CLIENT_ID", "").strip()
+        self.sp_client_secret = os.getenv("SPOTIFY_CLIENT_SECRET", "").strip()
+        self.sp_access_token = None
+
+    def _normalize_name(self, name: str) -> str:
+        return re.sub(r"[^a-z0-9]", "", str(name).lower())
+
+    def _extract_yt_id(self, url: str) -> Optional[str]:
+        m = re.search(r"channel/(UC[0-9A-Za-z_-]{22})", str(url))
+        return m.group(1) if m else None
+
+    def _extract_sp_id(self, url: str) -> Optional[str]:
+        m = re.search(r"artist/([0-9A-Za-z]{22})", str(url))
+        return m.group(1) if m else None
+
+    # =========================================================================
+    # 1. SPOTIFY AUTH & BATCH API
+    # =========================================================================
+    def authenticate_spotify(self) -> bool:
+        if not self.sp_client_id or not self.sp_client_secret:
+            logger.warning("Spotify credentials missing in .env")
+            return False
+
+        url = "https://accounts.spotify.com/api/token"
+        auth_bytes = f"{self.sp_client_id}:{self.sp_client_secret}".encode("utf-8")
+        headers = {
+            "Authorization": f"Basic {base64.b64encode(auth_bytes).decode('utf-8')}",
+            "Content-Type": "application/x-www-form-urlencoded"
+        }
+        data = {"grant_type": "client_credentials"}
+        try:
+            resp = requests.post(url, headers=headers, data=data, timeout=10)
+            if resp.status_code == 200:
+                self.sp_access_token = resp.json().get("access_token")
+                logger.info("Successfully authenticated with Spotify Web API.")
+                return True
+            else:
+                logger.error(f"Spotify authentication failed: HTTP {resp.status_code} - {resp.text}")
+                return False
+        except Exception as e:
+            logger.error(f"Spotify authentication error: {e}")
+            return False
+
+    def fetch_spotify_api_batch(self, sp_ids: List[str]) -> Dict[str, Dict]:
+        results = {}
+        if not self.sp_access_token or not sp_ids:
+            return results
+
+        headers = {"Authorization": f"Bearer {self.sp_access_token}"}
+        chunk_size = 50
+        for i in range(0, len(sp_ids), chunk_size):
+            chunk = sp_ids[i:i + chunk_size]
+            url = f"https://api.spotify.com/v1/artists?ids={','.join(chunk)}"
+            try:
+                resp = requests.get(url, headers=headers, timeout=12)
+                if resp.status_code == 200:
+                    for art in resp.json().get("artists", []):
+                        if art:
+                            results[art["id"]] = {
+                                "followers": art.get("followers", {}).get("total", 0),
+                                "popularity": art.get("popularity", 0)
+                            }
+                else:
+                    logger.warning(f"Spotify batch failed: HTTP {resp.status_code}")
+            except Exception as e:
+                logger.error(f"Error fetching Spotify batch: {e}")
+            time.sleep(0.05)
+        return results
+
+    # =========================================================================
+    # 2. SPOTIFY KWORB MONTHLY LISTENERS INDEX
+    # =========================================================================
+    def fetch_kworb_listeners(self) -> Dict[str, Tuple[int, int]]:
+        url = "https://kworb.net/spotify/listeners.html"
+        logger.info(f"Fetching global monthly listener index from {url}...")
+        results = {}
+        try:
+            resp = self.session.get(url, timeout=15)
+            if resp.status_code == 200:
+                soup = BeautifulSoup(resp.text, "html.parser")
+                table = soup.find("table")
+                if table:
+                    for tr in table.find_all("tr"):
+                        tds = tr.find_all("td")
+                        if len(tds) >= 4 and tds[1].find("a"):
+                            name = tds[1].find("a").get_text(strip=True)
+                            raw_l = tds[2].get_text(strip=True).replace(",", "")
+                            raw_c = tds[3].get_text(strip=True).replace(",", "").replace("+", "").replace("−", "-")
+                            try:
+                                listeners = int(raw_l)
+                                change = int(raw_c) if raw_c.lstrip("-").isdigit() else 0
+                                results[self._normalize_name(name)] = (listeners, change)
+                            except ValueError:
+                                continue
+                logger.info(f"Loaded {len(results)} artists from Kworb monthly listeners index.")
+        except Exception as e:
+            logger.error(f"Error loading Kworb table: {e}")
+        return results
+
+    # =========================================================================
+    # 3. YOUTUBE DATA API BATCH (CHANNELS.LIST)
+    # =========================================================================
+    def fetch_youtube_api_batch(self, channel_ids: List[str]) -> Dict[str, Tuple[int, int]]:
+        results = {}
+        if not self.yt_api_key:
+            logger.warning("YOUTUBE_API_KEY missing in .env")
+            return results
+
+        logger.info(f"Querying YouTube Data API v3 for {len(channel_ids)} channels in batches of 50...")
+        chunk_size = 50
+        for i in range(0, len(channel_ids), chunk_size):
+            chunk = channel_ids[i:i + chunk_size]
+            url = f"https://www.googleapis.com/youtube/v3/channels?part=statistics&id={','.join(chunk)}&key={self.yt_api_key}"
+            try:
+                resp = requests.get(url, timeout=12)
+                if resp.status_code == 200:
+                    for item in resp.json().get("items", []):
+                        cid = item["id"]
+                        stats = item.get("statistics", {})
+                        subs = int(stats.get("subscriberCount", 0))
+                        views = int(stats.get("viewCount", 0))
+                        results[cid] = (subs, views)
+                else:
+                    logger.warning(f"YouTube API returned HTTP {resp.status_code}: {resp.text[:120]}")
+            except Exception as e:
+                logger.error(f"Error in YouTube batch call: {e}")
+            time.sleep(0.05)
+        logger.info(f"YouTube Data API successfully resolved {len(results)} / {len(channel_ids)} channels.")
+        return results
+
+    # =========================================================================
+    # 4. MASTER SYNCHRONIZATION
+    # =========================================================================
+    def run_master_sync(self):
+        conn = sqlite3.connect(self.db_path)
+        cur = conn.cursor()
+
+        cur.execute("""
+            SELECT a.artist_id, a.artist_name, a.youtube_url, a.spotify_url,
+                   t.subscribers_count, t.spotify_monthly_listeners
+            FROM artists a
+            JOIN telemetry_metrics t ON a.artist_id = t.artist_id
+            WHERE a.image_verified_flag = 1
+        """)
+        rows = cur.fetchall()
+        logger.info(f"Loaded {len(rows)} verified cohort records from database.")
+
+        # Collect Channel IDs and Spotify IDs
+        yt_ids = [self._extract_yt_id(r[2]) for r in rows if self._extract_yt_id(r[2])]
+        sp_ids = [self._extract_sp_id(r[3]) for r in rows if self._extract_sp_id(r[3])]
+
+        # Execute Batch Platform Ingestions
+        yt_metrics = self.fetch_youtube_api_batch(yt_ids)
+        
+        has_sp_api = self.authenticate_spotify()
+        sp_api_metrics = self.fetch_spotify_api_batch(sp_ids) if has_sp_api else {}
+        kworb_metrics = self.fetch_kworb_listeners()
+
+        # Update Database Atomically
+        now_iso = datetime.now(timezone.utc).isoformat()
+        synced_yt = 0
+        synced_sp = 0
+
+        for artist_id, name, yt_url, sp_url, curr_subs, curr_listeners in rows:
+            updates = ["last_synced_at = ?"]
+            params = [now_iso]
+
+            # 1. Update YouTube Data API Metrics
+            cid = self._extract_yt_id(yt_url)
+            if cid and cid in yt_metrics:
+                subs, views = yt_metrics[cid]
+                if subs > 0:
+                    updates.append("subscribers_count = ?")
+                    params.append(subs)
+                    updates.append("total_views = ?")
+                    params.append(views)
+                    synced_yt += 1
+
+            # 2. Update Spotify Monthly Listeners (from Kworb Index)
+            norm = self._normalize_name(name)
+            if norm in kworb_metrics:
+                listeners, change = kworb_metrics[norm]
+                updates.append("spotify_monthly_listeners = ?")
+                params.append(listeners)
+                updates.append("spotify_daily_change = ?")
+                params.append(change)
+                synced_sp += 1
+            else:
+                sid = self._extract_sp_id(sp_url)
+                if sid and sid in sp_api_metrics:
+                    fol = sp_api_metrics[sid]["followers"]
+                    if (curr_listeners is None or curr_listeners < 1000) and fol > 1000:
+                        updates.append("spotify_monthly_listeners = ?")
+                        params.append(fol)
+                        synced_sp += 1
+
+            if len(updates) > 1:
+                params.append(artist_id)
+                cur.execute(f"UPDATE telemetry_metrics SET {', '.join(updates)} WHERE artist_id = ?", params)
+
+        conn.commit()
+        conn.close()
+
+        logger.info("=" * 60)
+        logger.info("MASTER SYNCHRONIZATION COMPLETED SUCCESSFULLY")
+        logger.info(f"  Live YouTube Channels Updated (API v3): {synced_yt} / {len(rows)}")
+        logger.info(f"  Verified Spotify Telemetry Updated:     {synced_sp} / {len(rows)}")
+        logger.info("=" * 60)
+
+
+if __name__ == "__main__":
+    sync = TelemetryMasterSync()
+    sync.run_master_sync()