"""
collectors/sync_complete_catalog.py
Master Production Telemetry Synchronizer for HIFFI Catalog.
Reads credentials automatically from .env.
- YouTube Data API v3 (Batch 50 IDs per call)
- Spotify Web API Client Credentials (Batch 50 IDs per call)
- Kworb Global Index for Spotify Monthly Listeners
"""

import os
import re
import time
import base64
import sqlite3
import logging
from pathlib import Path
from datetime import datetime, timezone
from typing import Dict, List, Optional, Tuple
import requests
from bs4 import BeautifulSoup
from dotenv import load_dotenv

# Load .env from project root
PROJECT_ROOT = Path(__file__).resolve().parent.parent
load_dotenv(PROJECT_ROOT / ".env")

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s"
)
logger = logging.getLogger("TelemetryMaster")

DB_PATH = PROJECT_ROOT / "data" / "hiffi_catalog.db"

HTTP_HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
}


class TelemetryMasterSync:
    def __init__(self, db_path: Path = DB_PATH):
        self.db_path = db_path
        self.session = requests.Session()
        self.session.headers.update(HTTP_HEADERS)
        
        self.yt_api_key = os.getenv("YOUTUBE_API_KEY", "").strip()
        self.sp_client_id = os.getenv("SPOTIFY_CLIENT_ID", "").strip()
        self.sp_client_secret = os.getenv("SPOTIFY_CLIENT_SECRET", "").strip()
        self.sp_access_token = None

    def _normalize_name(self, name: str) -> str:
        return re.sub(r"[^a-z0-9]", "", str(name).lower())

    def _extract_yt_id(self, url: str) -> Optional[str]:
        m = re.search(r"channel/(UC[0-9A-Za-z_-]{22})", str(url))
        return m.group(1) if m else None

    def _extract_sp_id(self, url: str) -> Optional[str]:
        m = re.search(r"artist/([0-9A-Za-z]{22})", str(url))
        return m.group(1) if m else None

    # =========================================================================
    # 1. SPOTIFY AUTH & BATCH API
    # =========================================================================
    def authenticate_spotify(self) -> bool:
        if not self.sp_client_id or not self.sp_client_secret:
            logger.warning("Spotify credentials missing in .env")
            return False

        url = "https://accounts.spotify.com/api/token"
        auth_bytes = f"{self.sp_client_id}:{self.sp_client_secret}".encode("utf-8")
        headers = {
            "Authorization": f"Basic {base64.b64encode(auth_bytes).decode('utf-8')}",
            "Content-Type": "application/x-www-form-urlencoded"
        }
        data = {"grant_type": "client_credentials"}
        try:
            resp = requests.post(url, headers=headers, data=data, timeout=10)
            if resp.status_code == 200:
                self.sp_access_token = resp.json().get("access_token")
                logger.info("Successfully authenticated with Spotify Web API.")
                return True
            else:
                logger.error(f"Spotify authentication failed: HTTP {resp.status_code} - {resp.text}")
                return False
        except Exception as e:
            logger.error(f"Spotify authentication error: {e}")
            return False

    def fetch_spotify_api_batch(self, sp_ids: List[str]) -> Dict[str, Dict]:
        results = {}
        if not self.sp_access_token or not sp_ids:
            return results

        headers = {"Authorization": f"Bearer {self.sp_access_token}"}
        chunk_size = 50
        for i in range(0, len(sp_ids), chunk_size):
            chunk = sp_ids[i:i + chunk_size]
            url = f"https://api.spotify.com/v1/artists?ids={','.join(chunk)}"
            try:
                resp = requests.get(url, headers=headers, timeout=12)
                if resp.status_code == 200:
                    for art in resp.json().get("artists", []):
                        if art:
                            results[art["id"]] = {
                                "followers": art.get("followers", {}).get("total", 0),
                                "popularity": art.get("popularity", 0)
                            }
                else:
                    logger.warning(f"Spotify batch failed: HTTP {resp.status_code}")
            except Exception as e:
                logger.error(f"Error fetching Spotify batch: {e}")
            time.sleep(0.05)
        return results

    # =========================================================================
    # 2. SPOTIFY KWORB MONTHLY LISTENERS INDEX
    # =========================================================================
    def fetch_kworb_listeners(self) -> Dict[str, Tuple[int, int]]:
        url = "https://kworb.net/spotify/listeners.html"
        logger.info(f"Fetching global monthly listener index from {url}...")
        results = {}
        try:
            resp = self.session.get(url, timeout=15)
            if resp.status_code == 200:
                soup = BeautifulSoup(resp.text, "html.parser")
                table = soup.find("table")
                if table:
                    for tr in table.find_all("tr"):
                        tds = tr.find_all("td")
                        if len(tds) >= 4 and tds[1].find("a"):
                            name = tds[1].find("a").get_text(strip=True)
                            raw_l = tds[2].get_text(strip=True).replace(",", "")
                            raw_c = tds[3].get_text(strip=True).replace(",", "").replace("+", "").replace("−", "-")
                            try:
                                listeners = int(raw_l)
                                change = int(raw_c) if raw_c.lstrip("-").isdigit() else 0
                                results[self._normalize_name(name)] = (listeners, change)
                            except ValueError:
                                continue
                logger.info(f"Loaded {len(results)} artists from Kworb monthly listeners index.")
        except Exception as e:
            logger.error(f"Error loading Kworb table: {e}")
        return results

    # =========================================================================
    # 3. YOUTUBE DATA API BATCH (CHANNELS.LIST)
    # =========================================================================
    def fetch_youtube_api_batch(self, channel_ids: List[str]) -> Dict[str, Tuple[int, int]]:
        results = {}
        if not self.yt_api_key:
            logger.warning("YOUTUBE_API_KEY missing in .env")
            return results

        logger.info(f"Querying YouTube Data API v3 for {len(channel_ids)} channels in batches of 50...")
        chunk_size = 50
        for i in range(0, len(channel_ids), chunk_size):
            chunk = channel_ids[i:i + chunk_size]
            url = f"https://www.googleapis.com/youtube/v3/channels?part=statistics&id={','.join(chunk)}&key={self.yt_api_key}"
            try:
                resp = requests.get(url, timeout=12)
                if resp.status_code == 200:
                    for item in resp.json().get("items", []):
                        cid = item["id"]
                        stats = item.get("statistics", {})
                        subs = int(stats.get("subscriberCount", 0))
                        views = int(stats.get("viewCount", 0))
                        results[cid] = (subs, views)
                else:
                    logger.warning(f"YouTube API returned HTTP {resp.status_code}: {resp.text[:120]}")
            except Exception as e:
                logger.error(f"Error in YouTube batch call: {e}")
            time.sleep(0.05)
        logger.info(f"YouTube Data API successfully resolved {len(results)} / {len(channel_ids)} channels.")
        return results

    # =========================================================================
    # 4. MASTER SYNCHRONIZATION
    # =========================================================================
    def run_master_sync(self):
        conn = sqlite3.connect(self.db_path)
        cur = conn.cursor()

        cur.execute("""
            SELECT a.artist_id, a.artist_name, a.youtube_url, a.spotify_url,
                   t.subscribers_count, t.spotify_monthly_listeners
            FROM artists a
            JOIN telemetry_metrics t ON a.artist_id = t.artist_id
            WHERE a.image_verified_flag = 1
        """)
        rows = cur.fetchall()
        logger.info(f"Loaded {len(rows)} verified cohort records from database.")

        # Collect Channel IDs and Spotify IDs
        yt_ids = [self._extract_yt_id(r[2]) for r in rows if self._extract_yt_id(r[2])]
        sp_ids = [self._extract_sp_id(r[3]) for r in rows if self._extract_sp_id(r[3])]

        # Execute Batch Platform Ingestions
        yt_metrics = self.fetch_youtube_api_batch(yt_ids)
        
        has_sp_api = self.authenticate_spotify()
        sp_api_metrics = self.fetch_spotify_api_batch(sp_ids) if has_sp_api else {}
        kworb_metrics = self.fetch_kworb_listeners()

        # Update Database Atomically
        now_iso = datetime.now(timezone.utc).isoformat()
        synced_yt = 0
        synced_sp = 0

        for artist_id, name, yt_url, sp_url, curr_subs, curr_listeners in rows:
            updates = ["last_synced_at = ?"]
            params = [now_iso]

            # 1. Update YouTube Data API Metrics
            cid = self._extract_yt_id(yt_url)
            if cid and cid in yt_metrics:
                subs, views = yt_metrics[cid]
                if subs > 0:
                    updates.append("subscribers_count = ?")
                    params.append(subs)
                    updates.append("total_views = ?")
                    params.append(views)
                    synced_yt += 1

            # 2. Update Spotify Monthly Listeners (from Kworb Index)
            norm = self._normalize_name(name)
            if norm in kworb_metrics:
                listeners, change = kworb_metrics[norm]
                updates.append("spotify_monthly_listeners = ?")
                params.append(listeners)
                updates.append("spotify_daily_change = ?")
                params.append(change)
                synced_sp += 1
            else:
                sid = self._extract_sp_id(sp_url)
                if sid and sid in sp_api_metrics:
                    fol = sp_api_metrics[sid]["followers"]
                    if (curr_listeners is None or curr_listeners < 1000) and fol > 1000:
                        updates.append("spotify_monthly_listeners = ?")
                        params.append(fol)
                        synced_sp += 1

            if len(updates) > 1:
                params.append(artist_id)
                cur.execute(f"UPDATE telemetry_metrics SET {', '.join(updates)} WHERE artist_id = ?", params)

        conn.commit()
        conn.close()

        logger.info("=" * 60)
        logger.info("MASTER SYNCHRONIZATION COMPLETED SUCCESSFULLY")
        logger.info(f"  Live YouTube Channels Updated (API v3): {synced_yt} / {len(rows)}")
        logger.info(f"  Verified Spotify Telemetry Updated:     {synced_sp} / {len(rows)}")
        logger.info("=" * 60)


if __name__ == "__main__":
    sync = TelemetryMasterSync()
    sync.run_master_sync()