checkpoint: working 10-artist batch with log-normalized ranking

collectors/youtube_collector.py

@@ -0,0 +1,192 @@
+import os
+import re
+import json
+import sqlite3
+from pathlib import Path
+from datetime import datetime, timezone, timedelta
+from urllib.parse import urlparse
+from typing import Dict, Any, Optional
+
+try:
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+
+class YouTubeCollector:
+    def __init__(self, api_keys: list[str] = None, cache_db: str = ".cache/youtube_metrics.sqlite"):
+        self.api_keys = [k.strip() for k in (api_keys or []) if k.strip()]
+        self.cache_db = Path(cache_db)
+        self.cache_db.parent.mkdir(parents=True, exist_ok=True)
+        self._init_db()
+        self.client = None
+        if self.api_keys and build:
+            try:
+                self.client = build("youtube", "v3", developerKey=self.api_keys[0], cache_discovery=False)
+            except Exception as e:
+                print(f"[Warning] Failed to initialize YouTube API client: {e}")
+
+    def _init_db(self):
+        with sqlite3.connect(self.cache_db) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS channel_metrics (
+                    channel_id TEXT PRIMARY KEY,
+                    title TEXT,
+                    subscribers INTEGER,
+                    lifetime_views INTEGER,
+                    video_count INTEGER,
+                    recent_avg_views INTEGER,
+                    momentum_30d REAL,
+                    raw_json TEXT,
+                    updated_at TEXT
+                )
+            """)
+
+    def _get_cached(self, channel_id: str) -> Optional[Dict[str, Any]]:
+        with sqlite3.connect(self.cache_db) as conn:
+            conn.row_factory = sqlite3.Row
+            row = conn.execute("SELECT * FROM channel_metrics WHERE channel_id = ?", (channel_id,)).fetchone()
+            if row:
+                return dict(row)
+        return None
+
+    def _save_cache(self, metrics: Dict[str, Any]):
+        with sqlite3.connect(self.cache_db) as conn:
+            conn.execute("""
+                INSERT OR REPLACE INTO channel_metrics (
+                    channel_id, title, subscribers, lifetime_views, video_count,
+                    recent_avg_views, momentum_30d, raw_json, updated_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (
+                metrics["channel_id"],
+                metrics["title"],
+                metrics["subscribers"],
+                metrics["lifetime_views"],
+                metrics["video_count"],
+                metrics["recent_avg_views"],
+                metrics["momentum_30d"],
+                json.dumps(metrics),
+                datetime.now(timezone.utc).isoformat()
+            ))
+
+    def extract_channel_id(self, url: str) -> str:
+        """Extract UC... channel ID from standard YouTube URL."""
+        if not url:
+            return ""
+        match = re.search(r"channel/(UC[a-zA-Z0-9_-]{22})", url)
+        if match:
+            return match.group(1)
+        return url.strip().rstrip("/").split("/")[-1]
+
+    def get_channel_metrics(self, youtube_url: str, fetch_recent_videos: bool = False) -> Dict[str, Any]:
+        channel_id = self.extract_channel_id(youtube_url)
+        empty = {
+            "channel_id": channel_id,
+            "title": "",
+            "subscribers": 0,
+            "lifetime_views": 0,
+            "video_count": 0,
+            "recent_avg_views": 0,
+            "momentum_30d": 0.0,
+            "status": "unresolved"
+        }
+        if not channel_id:
+            empty["status"] = "empty_url"
+            return empty
+
+        # 1. Check local cache first (saves API quota)
+        cached = self._get_cached(channel_id)
+        if cached:
+            cached["status"] = "cached"
+            return cached
+
+        # 2. If client is not available (zero API keys), return baseline
+        if not self.client:
+            empty["status"] = "no_api_key"
+            return empty
+
+        try:
+            # 3. Call channels.list for statistics and contentDetails
+            resp = self.client.channels().list(
+                part="snippet,statistics,contentDetails",
+                id=channel_id
+            ).execute()
+
+            items = resp.get("items") or []
+            if not items:
+                empty["status"] = "not_found"
+                return empty
+
+            item = items[0]
+            snippet = item.get("snippet", {})
+            stats = item.get("statistics", {})
+            content = item.get("contentDetails", {})
+
+            metrics = {
+                "channel_id": channel_id,
+                "title": snippet.get("title", ""),
+                "subscribers": int(stats.get("subscriberCount") or 0),
+                "lifetime_views": int(stats.get("viewCount") or 0),
+                "video_count": int(stats.get("videoCount") or 0),
+                "recent_avg_views": 0,
+                "momentum_30d": 0.0,
+                "status": "live_api"
+            }
+
+            # 4. Optional: fetch recent video metrics for average views & momentum
+            uploads_id = content.get("relatedPlaylists", {}).get("uploads")
+            if fetch_recent_videos and uploads_id:
+                recent_views, momentum = self._calc_recent_metrics(uploads_id, metrics["lifetime_views"])
+                metrics["recent_avg_views"] = recent_views
+                metrics["momentum_30d"] = momentum
+
+            self._save_cache(metrics)
+            return metrics
+
+        except Exception as e:
+            empty["status"] = f"error: {str(e)[:100]}"
+            return empty
+
+    def _calc_recent_metrics(self, uploads_playlist_id: str, lifetime_views: int) -> tuple[int, float]:
+        """Fetch up to 10 latest uploads to compute average views and 30d momentum."""
+        try:
+            playlist_resp = self.client.playlistItems().list(
+                part="contentDetails",
+                playlistId=uploads_playlist_id,
+                maxResults=10
+            ).execute()
+
+            video_ids = [i["contentDetails"]["videoId"] for i in playlist_resp.get("items", [])]
+            if not video_ids:
+                return 0, 0.0
+
+            video_resp = self.client.videos().list(
+                part="statistics,snippet",
+                id=",".join(video_ids)
+            ).execute()
+
+            now = datetime.now(timezone.utc)
+            thirty_days_ago = now - timedelta(days=30)
+            
+            views = []
+            views_30d = 0
+
+            for v in video_resp.get("items", []):
+                v_views = int(v.get("statistics", {}).get("viewCount") or 0)
+                views.append(v_views)
+                pub_date_str = v.get("snippet", {}).get("publishedAt", "")
+                if pub_date_str:
+                    pub_date = datetime.fromisoformat(pub_date_str.replace("Z", "+00:00"))
+                    if pub_date >= thirty_days_ago:
+                        views_30d += v_views
+
+            avg_views = int(sum(views) / len(views)) if views else 0
+            momentum = round((views_30d / lifetime_views) * 100, 3) if lifetime_views > 0 else 0.0
+            return avg_views, momentum
+        except Exception:
+            return 0, 0.0
+
+if __name__ == "__main__":
+    collector = YouTubeCollector()
+    test_res = collector.get_channel_metrics("https://www.youtube.com/channel/UCBOxSirnH1-_-xKtqX051lg")
+    print("Test run result:")
+    print(test_res)