259 lines
11 KiB
Python
259 lines
11 KiB
Python
|
"""
|
|||
|
|
collectors/sync_complete_catalog.py
|
|||
|
|
Master Production Telemetry Synchronizer for HIFFI Catalog.
|
|||
|
|
Reads credentials automatically from .env.
|
|||
|
|
- YouTube Data API v3 (Batch 50 IDs per call)
|
|||
|
|
- Spotify Web API Client Credentials (Batch 50 IDs per call)
|
|||
|
|
- Kworb Global Index for Spotify Monthly Listeners
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
import time
|
|||
|
|
import base64
|
|||
|
|
import sqlite3
|
|||
|
|
import logging
|
|||
|
|
from pathlib import Path
|
|||
|
|
from datetime import datetime, timezone
|
|||
|
|
from typing import Dict, List, Optional, Tuple
|
|||
|
|
import requests
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
from dotenv import load_dotenv
|
|||
|
|
|
|||
|
|
# Load .env from project root
|
|||
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|||
|
|
load_dotenv(PROJECT_ROOT / ".env")
|
|||
|
|
|
|||
|
|
logging.basicConfig(
|
|||
|
|
level=logging.INFO,
|
|||
|
|
format="%(asctime)s [%(levelname)s] %(message)s"
|
|||
|
|
)
|
|||
|
|
logger = logging.getLogger("TelemetryMaster")
|
|||
|
|
|
|||
|
|
DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"
|
|||
|
|
|
|||
|
|
HTTP_HEADERS = {
|
|||
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
class TelemetryMasterSync:
|
|||
|
|
def __init__(self, db_path: Path = DB_PATH):
|
|||
|
|
self.db_path = db_path
|
|||
|
|
self.session = requests.Session()
|
|||
|
|
self.session.headers.update(HTTP_HEADERS)
|
|||
|
|
|
|||
|
|
self.yt_api_key = os.getenv("YOUTUBE_API_KEY", "").strip()
|
|||
|
|
self.sp_client_id = os.getenv("SPOTIFY_CLIENT_ID", "").strip()
|
|||
|
|
self.sp_client_secret = os.getenv("SPOTIFY_CLIENT_SECRET", "").strip()
|
|||
|
|
self.sp_access_token = None
|
|||
|
|
|
|||
|
|
def _normalize_name(self, name: str) -> str:
|
|||
|
|
return re.sub(r"[^a-z0-9]", "", str(name).lower())
|
|||
|
|
|
|||
|
|
def _extract_yt_id(self, url: str) -> Optional[str]:
|
|||
|
|
m = re.search(r"channel/(UC[0-9A-Za-z_-]{22})", str(url))
|
|||
|
|
return m.group(1) if m else None
|
|||
|
|
|
|||
|
|
def _extract_sp_id(self, url: str) -> Optional[str]:
|
|||
|
|
m = re.search(r"artist/([0-9A-Za-z]{22})", str(url))
|
|||
|
|
return m.group(1) if m else None
|
|||
|
|
|
|||
|
|
# =========================================================================
|
|||
|
|
# 1. SPOTIFY AUTH & BATCH API
|
|||
|
|
# =========================================================================
|
|||
|
|
def authenticate_spotify(self) -> bool:
|
|||
|
|
if not self.sp_client_id or not self.sp_client_secret:
|
|||
|
|
logger.warning("Spotify credentials missing in .env")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
url = "https://accounts.spotify.com/api/token"
|
|||
|
|
auth_bytes = f"{self.sp_client_id}:{self.sp_client_secret}".encode("utf-8")
|
|||
|
|
headers = {
|
|||
|
|
"Authorization": f"Basic {base64.b64encode(auth_bytes).decode('utf-8')}",
|
|||
|
|
"Content-Type": "application/x-www-form-urlencoded"
|
|||
|
|
}
|
|||
|
|
data = {"grant_type": "client_credentials"}
|
|||
|
|
try:
|
|||
|
|
resp = requests.post(url, headers=headers, data=data, timeout=10)
|
|||
|
|
if resp.status_code == 200:
|
|||
|
|
self.sp_access_token = resp.json().get("access_token")
|
|||
|
|
logger.info("Successfully authenticated with Spotify Web API.")
|
|||
|
|
return True
|
|||
|
|
else:
|
|||
|
|
logger.error(f"Spotify authentication failed: HTTP {resp.status_code} - {resp.text}")
|
|||
|
|
return False
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"Spotify authentication error: {e}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
def fetch_spotify_api_batch(self, sp_ids: List[str]) -> Dict[str, Dict]:
|
|||
|
|
results = {}
|
|||
|
|
if not self.sp_access_token or not sp_ids:
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
headers = {"Authorization": f"Bearer {self.sp_access_token}"}
|
|||
|
|
chunk_size = 50
|
|||
|
|
for i in range(0, len(sp_ids), chunk_size):
|
|||
|
|
chunk = sp_ids[i:i + chunk_size]
|
|||
|
|
url = f"https://api.spotify.com/v1/artists?ids={','.join(chunk)}"
|
|||
|
|
try:
|
|||
|
|
resp = requests.get(url, headers=headers, timeout=12)
|
|||
|
|
if resp.status_code == 200:
|
|||
|
|
for art in resp.json().get("artists", []):
|
|||
|
|
if art:
|
|||
|
|
results[art["id"]] = {
|
|||
|
|
"followers": art.get("followers", {}).get("total", 0),
|
|||
|
|
"popularity": art.get("popularity", 0)
|
|||
|
|
}
|
|||
|
|
else:
|
|||
|
|
logger.warning(f"Spotify batch failed: HTTP {resp.status_code}")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"Error fetching Spotify batch: {e}")
|
|||
|
|
time.sleep(0.05)
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
# =========================================================================
|
|||
|
|
# 2. SPOTIFY KWORB MONTHLY LISTENERS INDEX
|
|||
|
|
# =========================================================================
|
|||
|
|
def fetch_kworb_listeners(self) -> Dict[str, Tuple[int, int]]:
|
|||
|
|
url = "https://kworb.net/spotify/listeners.html"
|
|||
|
|
logger.info(f"Fetching global monthly listener index from {url}...")
|
|||
|
|
results = {}
|
|||
|
|
try:
|
|||
|
|
resp = self.session.get(url, timeout=15)
|
|||
|
|
if resp.status_code == 200:
|
|||
|
|
soup = BeautifulSoup(resp.text, "html.parser")
|
|||
|
|
table = soup.find("table")
|
|||
|
|
if table:
|
|||
|
|
for tr in table.find_all("tr"):
|
|||
|
|
tds = tr.find_all("td")
|
|||
|
|
if len(tds) >= 4 and tds[1].find("a"):
|
|||
|
|
name = tds[1].find("a").get_text(strip=True)
|
|||
|
|
raw_l = tds[2].get_text(strip=True).replace(",", "")
|
|||
|
|
raw_c = tds[3].get_text(strip=True).replace(",", "").replace("+", "").replace("−", "-")
|
|||
|
|
try:
|
|||
|
|
listeners = int(raw_l)
|
|||
|
|
change = int(raw_c) if raw_c.lstrip("-").isdigit() else 0
|
|||
|
|
results[self._normalize_name(name)] = (listeners, change)
|
|||
|
|
except ValueError:
|
|||
|
|
continue
|
|||
|
|
logger.info(f"Loaded {len(results)} artists from Kworb monthly listeners index.")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"Error loading Kworb table: {e}")
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
# =========================================================================
|
|||
|
|
# 3. YOUTUBE DATA API BATCH (CHANNELS.LIST)
|
|||
|
|
# =========================================================================
|
|||
|
|
def fetch_youtube_api_batch(self, channel_ids: List[str]) -> Dict[str, Tuple[int, int]]:
|
|||
|
|
results = {}
|
|||
|
|
if not self.yt_api_key:
|
|||
|
|
logger.warning("YOUTUBE_API_KEY missing in .env")
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
logger.info(f"Querying YouTube Data API v3 for {len(channel_ids)} channels in batches of 50...")
|
|||
|
|
chunk_size = 50
|
|||
|
|
for i in range(0, len(channel_ids), chunk_size):
|
|||
|
|
chunk = channel_ids[i:i + chunk_size]
|
|||
|
|
url = f"https://www.googleapis.com/youtube/v3/channels?part=statistics&id={','.join(chunk)}&key={self.yt_api_key}"
|
|||
|
|
try:
|
|||
|
|
resp = requests.get(url, timeout=12)
|
|||
|
|
if resp.status_code == 200:
|
|||
|
|
for item in resp.json().get("items", []):
|
|||
|
|
cid = item["id"]
|
|||
|
|
stats = item.get("statistics", {})
|
|||
|
|
subs = int(stats.get("subscriberCount", 0))
|
|||
|
|
views = int(stats.get("viewCount", 0))
|
|||
|
|
results[cid] = (subs, views)
|
|||
|
|
else:
|
|||
|
|
logger.warning(f"YouTube API returned HTTP {resp.status_code}: {resp.text[:120]}")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"Error in YouTube batch call: {e}")
|
|||
|
|
time.sleep(0.05)
|
|||
|
|
logger.info(f"YouTube Data API successfully resolved {len(results)} / {len(channel_ids)} channels.")
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
# =========================================================================
|
|||
|
|
# 4. MASTER SYNCHRONIZATION
|
|||
|
|
# =========================================================================
|
|||
|
|
def run_master_sync(self):
|
|||
|
|
conn = sqlite3.connect(self.db_path)
|
|||
|
|
cur = conn.cursor()
|
|||
|
|
|
|||
|
|
cur.execute("""
|
|||
|
|
SELECT a.artist_id, a.artist_name, a.youtube_url, a.spotify_url,
|
|||
|
|
t.subscribers_count, t.spotify_monthly_listeners
|
|||
|
|
FROM artists a
|
|||
|
|
JOIN telemetry_metrics t ON a.artist_id = t.artist_id
|
|||
|
|
WHERE a.image_verified_flag = 1
|
|||
|
|
""")
|
|||
|
|
rows = cur.fetchall()
|
|||
|
|
logger.info(f"Loaded {len(rows)} verified cohort records from database.")
|
|||
|
|
|
|||
|
|
# Collect Channel IDs and Spotify IDs
|
|||
|
|
yt_ids = [self._extract_yt_id(r[2]) for r in rows if self._extract_yt_id(r[2])]
|
|||
|
|
sp_ids = [self._extract_sp_id(r[3]) for r in rows if self._extract_sp_id(r[3])]
|
|||
|
|
|
|||
|
|
# Execute Batch Platform Ingestions
|
|||
|
|
yt_metrics = self.fetch_youtube_api_batch(yt_ids)
|
|||
|
|
|
|||
|
|
has_sp_api = self.authenticate_spotify()
|
|||
|
|
sp_api_metrics = self.fetch_spotify_api_batch(sp_ids) if has_sp_api else {}
|
|||
|
|
kworb_metrics = self.fetch_kworb_listeners()
|
|||
|
|
|
|||
|
|
# Update Database Atomically
|
|||
|
|
now_iso = datetime.now(timezone.utc).isoformat()
|
|||
|
|
synced_yt = 0
|
|||
|
|
synced_sp = 0
|
|||
|
|
|
|||
|
|
for artist_id, name, yt_url, sp_url, curr_subs, curr_listeners in rows:
|
|||
|
|
updates = ["last_synced_at = ?"]
|
|||
|
|
params = [now_iso]
|
|||
|
|
|
|||
|
|
# 1. Update YouTube Data API Metrics
|
|||
|
|
cid = self._extract_yt_id(yt_url)
|
|||
|
|
if cid and cid in yt_metrics:
|
|||
|
|
subs, views = yt_metrics[cid]
|
|||
|
|
if subs > 0:
|
|||
|
|
updates.append("subscribers_count = ?")
|
|||
|
|
params.append(subs)
|
|||
|
|
updates.append("total_views = ?")
|
|||
|
|
params.append(views)
|
|||
|
|
synced_yt += 1
|
|||
|
|
|
|||
|
|
# 2. Update Spotify Monthly Listeners (from Kworb Index)
|
|||
|
|
norm = self._normalize_name(name)
|
|||
|
|
if norm in kworb_metrics:
|
|||
|
|
listeners, change = kworb_metrics[norm]
|
|||
|
|
updates.append("spotify_monthly_listeners = ?")
|
|||
|
|
params.append(listeners)
|
|||
|
|
updates.append("spotify_daily_change = ?")
|
|||
|
|
params.append(change)
|
|||
|
|
synced_sp += 1
|
|||
|
|
else:
|
|||
|
|
sid = self._extract_sp_id(sp_url)
|
|||
|
|
if sid and sid in sp_api_metrics:
|
|||
|
|
fol = sp_api_metrics[sid]["followers"]
|
|||
|
|
if (curr_listeners is None or curr_listeners < 1000) and fol > 1000:
|
|||
|
|
updates.append("spotify_monthly_listeners = ?")
|
|||
|
|
params.append(fol)
|
|||
|
|
synced_sp += 1
|
|||
|
|
|
|||
|
|
if len(updates) > 1:
|
|||
|
|
params.append(artist_id)
|
|||
|
|
cur.execute(f"UPDATE telemetry_metrics SET {', '.join(updates)} WHERE artist_id = ?", params)
|
|||
|
|
|
|||
|
|
conn.commit()
|
|||
|
|
conn.close()
|
|||
|
|
|
|||
|
|
logger.info("=" * 60)
|
|||
|
|
logger.info("MASTER SYNCHRONIZATION COMPLETED SUCCESSFULLY")
|
|||
|
|
logger.info(f" Live YouTube Channels Updated (API v3): {synced_yt} / {len(rows)}")
|
|||
|
|
logger.info(f" Verified Spotify Telemetry Updated: {synced_sp} / {len(rows)}")
|
|||
|
|
logger.info("=" * 60)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
sync = TelemetryMasterSync()
|
|||
|
|
sync.run_master_sync()
|