259 lines
11 KiB
Python
259 lines
11 KiB
Python
"""
|
||
collectors/sync_complete_catalog.py
|
||
Master Production Telemetry Synchronizer for HIFFI Catalog.
|
||
Reads credentials automatically from .env.
|
||
- YouTube Data API v3 (Batch 50 IDs per call)
|
||
- Spotify Web API Client Credentials (Batch 50 IDs per call)
|
||
- Kworb Global Index for Spotify Monthly Listeners
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import time
|
||
import base64
|
||
import sqlite3
|
||
import logging
|
||
from pathlib import Path
|
||
from datetime import datetime, timezone
|
||
from typing import Dict, List, Optional, Tuple
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
from dotenv import load_dotenv
|
||
|
||
# Load .env from project root
|
||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
load_dotenv(PROJECT_ROOT / ".env")
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(message)s"
|
||
)
|
||
logger = logging.getLogger("TelemetryMaster")
|
||
|
||
DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"
|
||
|
||
HTTP_HEADERS = {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||
}
|
||
|
||
|
||
class TelemetryMasterSync:
|
||
def __init__(self, db_path: Path = DB_PATH):
|
||
self.db_path = db_path
|
||
self.session = requests.Session()
|
||
self.session.headers.update(HTTP_HEADERS)
|
||
|
||
self.yt_api_key = os.getenv("YOUTUBE_API_KEY", "").strip()
|
||
self.sp_client_id = os.getenv("SPOTIFY_CLIENT_ID", "").strip()
|
||
self.sp_client_secret = os.getenv("SPOTIFY_CLIENT_SECRET", "").strip()
|
||
self.sp_access_token = None
|
||
|
||
def _normalize_name(self, name: str) -> str:
|
||
return re.sub(r"[^a-z0-9]", "", str(name).lower())
|
||
|
||
def _extract_yt_id(self, url: str) -> Optional[str]:
|
||
m = re.search(r"channel/(UC[0-9A-Za-z_-]{22})", str(url))
|
||
return m.group(1) if m else None
|
||
|
||
def _extract_sp_id(self, url: str) -> Optional[str]:
|
||
m = re.search(r"artist/([0-9A-Za-z]{22})", str(url))
|
||
return m.group(1) if m else None
|
||
|
||
# =========================================================================
|
||
# 1. SPOTIFY AUTH & BATCH API
|
||
# =========================================================================
|
||
def authenticate_spotify(self) -> bool:
|
||
if not self.sp_client_id or not self.sp_client_secret:
|
||
logger.warning("Spotify credentials missing in .env")
|
||
return False
|
||
|
||
url = "https://accounts.spotify.com/api/token"
|
||
auth_bytes = f"{self.sp_client_id}:{self.sp_client_secret}".encode("utf-8")
|
||
headers = {
|
||
"Authorization": f"Basic {base64.b64encode(auth_bytes).decode('utf-8')}",
|
||
"Content-Type": "application/x-www-form-urlencoded"
|
||
}
|
||
data = {"grant_type": "client_credentials"}
|
||
try:
|
||
resp = requests.post(url, headers=headers, data=data, timeout=10)
|
||
if resp.status_code == 200:
|
||
self.sp_access_token = resp.json().get("access_token")
|
||
logger.info("Successfully authenticated with Spotify Web API.")
|
||
return True
|
||
else:
|
||
logger.error(f"Spotify authentication failed: HTTP {resp.status_code} - {resp.text}")
|
||
return False
|
||
except Exception as e:
|
||
logger.error(f"Spotify authentication error: {e}")
|
||
return False
|
||
|
||
def fetch_spotify_api_batch(self, sp_ids: List[str]) -> Dict[str, Dict]:
|
||
results = {}
|
||
if not self.sp_access_token or not sp_ids:
|
||
return results
|
||
|
||
headers = {"Authorization": f"Bearer {self.sp_access_token}"}
|
||
chunk_size = 50
|
||
for i in range(0, len(sp_ids), chunk_size):
|
||
chunk = sp_ids[i:i + chunk_size]
|
||
url = f"https://api.spotify.com/v1/artists?ids={','.join(chunk)}"
|
||
try:
|
||
resp = requests.get(url, headers=headers, timeout=12)
|
||
if resp.status_code == 200:
|
||
for art in resp.json().get("artists", []):
|
||
if art:
|
||
results[art["id"]] = {
|
||
"followers": art.get("followers", {}).get("total", 0),
|
||
"popularity": art.get("popularity", 0)
|
||
}
|
||
else:
|
||
logger.warning(f"Spotify batch failed: HTTP {resp.status_code}")
|
||
except Exception as e:
|
||
logger.error(f"Error fetching Spotify batch: {e}")
|
||
time.sleep(0.05)
|
||
return results
|
||
|
||
# =========================================================================
|
||
# 2. SPOTIFY KWORB MONTHLY LISTENERS INDEX
|
||
# =========================================================================
|
||
def fetch_kworb_listeners(self) -> Dict[str, Tuple[int, int]]:
|
||
url = "https://kworb.net/spotify/listeners.html"
|
||
logger.info(f"Fetching global monthly listener index from {url}...")
|
||
results = {}
|
||
try:
|
||
resp = self.session.get(url, timeout=15)
|
||
if resp.status_code == 200:
|
||
soup = BeautifulSoup(resp.text, "html.parser")
|
||
table = soup.find("table")
|
||
if table:
|
||
for tr in table.find_all("tr"):
|
||
tds = tr.find_all("td")
|
||
if len(tds) >= 4 and tds[1].find("a"):
|
||
name = tds[1].find("a").get_text(strip=True)
|
||
raw_l = tds[2].get_text(strip=True).replace(",", "")
|
||
raw_c = tds[3].get_text(strip=True).replace(",", "").replace("+", "").replace("−", "-")
|
||
try:
|
||
listeners = int(raw_l)
|
||
change = int(raw_c) if raw_c.lstrip("-").isdigit() else 0
|
||
results[self._normalize_name(name)] = (listeners, change)
|
||
except ValueError:
|
||
continue
|
||
logger.info(f"Loaded {len(results)} artists from Kworb monthly listeners index.")
|
||
except Exception as e:
|
||
logger.error(f"Error loading Kworb table: {e}")
|
||
return results
|
||
|
||
# =========================================================================
|
||
# 3. YOUTUBE DATA API BATCH (CHANNELS.LIST)
|
||
# =========================================================================
|
||
def fetch_youtube_api_batch(self, channel_ids: List[str]) -> Dict[str, Tuple[int, int]]:
|
||
results = {}
|
||
if not self.yt_api_key:
|
||
logger.warning("YOUTUBE_API_KEY missing in .env")
|
||
return results
|
||
|
||
logger.info(f"Querying YouTube Data API v3 for {len(channel_ids)} channels in batches of 50...")
|
||
chunk_size = 50
|
||
for i in range(0, len(channel_ids), chunk_size):
|
||
chunk = channel_ids[i:i + chunk_size]
|
||
url = f"https://www.googleapis.com/youtube/v3/channels?part=statistics&id={','.join(chunk)}&key={self.yt_api_key}"
|
||
try:
|
||
resp = requests.get(url, timeout=12)
|
||
if resp.status_code == 200:
|
||
for item in resp.json().get("items", []):
|
||
cid = item["id"]
|
||
stats = item.get("statistics", {})
|
||
subs = int(stats.get("subscriberCount", 0))
|
||
views = int(stats.get("viewCount", 0))
|
||
results[cid] = (subs, views)
|
||
else:
|
||
logger.warning(f"YouTube API returned HTTP {resp.status_code}: {resp.text[:120]}")
|
||
except Exception as e:
|
||
logger.error(f"Error in YouTube batch call: {e}")
|
||
time.sleep(0.05)
|
||
logger.info(f"YouTube Data API successfully resolved {len(results)} / {len(channel_ids)} channels.")
|
||
return results
|
||
|
||
# =========================================================================
|
||
# 4. MASTER SYNCHRONIZATION
|
||
# =========================================================================
|
||
def run_master_sync(self):
|
||
conn = sqlite3.connect(self.db_path)
|
||
cur = conn.cursor()
|
||
|
||
cur.execute("""
|
||
SELECT a.artist_id, a.artist_name, a.youtube_url, a.spotify_url,
|
||
t.subscribers_count, t.spotify_monthly_listeners
|
||
FROM artists a
|
||
JOIN telemetry_metrics t ON a.artist_id = t.artist_id
|
||
WHERE a.image_verified_flag = 1
|
||
""")
|
||
rows = cur.fetchall()
|
||
logger.info(f"Loaded {len(rows)} verified cohort records from database.")
|
||
|
||
# Collect Channel IDs and Spotify IDs
|
||
yt_ids = [self._extract_yt_id(r[2]) for r in rows if self._extract_yt_id(r[2])]
|
||
sp_ids = [self._extract_sp_id(r[3]) for r in rows if self._extract_sp_id(r[3])]
|
||
|
||
# Execute Batch Platform Ingestions
|
||
yt_metrics = self.fetch_youtube_api_batch(yt_ids)
|
||
|
||
has_sp_api = self.authenticate_spotify()
|
||
sp_api_metrics = self.fetch_spotify_api_batch(sp_ids) if has_sp_api else {}
|
||
kworb_metrics = self.fetch_kworb_listeners()
|
||
|
||
# Update Database Atomically
|
||
now_iso = datetime.now(timezone.utc).isoformat()
|
||
synced_yt = 0
|
||
synced_sp = 0
|
||
|
||
for artist_id, name, yt_url, sp_url, curr_subs, curr_listeners in rows:
|
||
updates = ["last_synced_at = ?"]
|
||
params = [now_iso]
|
||
|
||
# 1. Update YouTube Data API Metrics
|
||
cid = self._extract_yt_id(yt_url)
|
||
if cid and cid in yt_metrics:
|
||
subs, views = yt_metrics[cid]
|
||
if subs > 0:
|
||
updates.append("subscribers_count = ?")
|
||
params.append(subs)
|
||
updates.append("total_views = ?")
|
||
params.append(views)
|
||
synced_yt += 1
|
||
|
||
# 2. Update Spotify Monthly Listeners (from Kworb Index)
|
||
norm = self._normalize_name(name)
|
||
if norm in kworb_metrics:
|
||
listeners, change = kworb_metrics[norm]
|
||
updates.append("spotify_monthly_listeners = ?")
|
||
params.append(listeners)
|
||
updates.append("spotify_daily_change = ?")
|
||
params.append(change)
|
||
synced_sp += 1
|
||
else:
|
||
sid = self._extract_sp_id(sp_url)
|
||
if sid and sid in sp_api_metrics:
|
||
fol = sp_api_metrics[sid]["followers"]
|
||
if (curr_listeners is None or curr_listeners < 1000) and fol > 1000:
|
||
updates.append("spotify_monthly_listeners = ?")
|
||
params.append(fol)
|
||
synced_sp += 1
|
||
|
||
if len(updates) > 1:
|
||
params.append(artist_id)
|
||
cur.execute(f"UPDATE telemetry_metrics SET {', '.join(updates)} WHERE artist_id = ?", params)
|
||
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
logger.info("=" * 60)
|
||
logger.info("MASTER SYNCHRONIZATION COMPLETED SUCCESSFULLY")
|
||
logger.info(f" Live YouTube Channels Updated (API v3): {synced_yt} / {len(rows)}")
|
||
logger.info(f" Verified Spotify Telemetry Updated: {synced_sp} / {len(rows)}")
|
||
logger.info("=" * 60)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sync = TelemetryMasterSync()
|
||
sync.run_master_sync() |