114 lines
3.3 KiB
Python
114 lines
3.3 KiB
Python
"""
|
|
scripts/scrape_live_youtube_151.py
|
|
Scrapes live, ground-truth subscriber counts, views, and videos directly
|
|
from YouTube for the 151 unresolved artists using canonical channel mappings.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
import requests
|
|
|
|
ROOT_DIR = Path(__file__).resolve().parent.parent
|
|
if str(ROOT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(ROOT_DIR))
|
|
|
|
from core.database import get_connection
|
|
|
|
MAP_FILE = ROOT_DIR / "data" / "youtube_channel_mappings.json"
|
|
|
|
HEADERS = {
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
|
),
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
}
|
|
|
|
|
|
def parse_metric(raw_text):
|
|
if not raw_text:
|
|
return 0
|
|
t = raw_text.upper().replace(",", "").strip()
|
|
match = re.search(r"([\d\.]+)\s*([KMB])?", t)
|
|
if not match:
|
|
return 0
|
|
val = float(match.group(1))
|
|
suffix = match.group(2)
|
|
if suffix == "K":
|
|
val *= 1e3
|
|
elif suffix == "M":
|
|
val *= 1e6
|
|
elif suffix == "B":
|
|
val *= 1e9
|
|
return int(val)
|
|
|
|
|
|
def main():
|
|
if not MAP_FILE.exists():
|
|
print(f"Mapping file not found at {MAP_FILE}")
|
|
return
|
|
|
|
with open(MAP_FILE, "r", encoding="utf-8") as f:
|
|
mappings = json.load(f)
|
|
|
|
conn = get_connection()
|
|
cur = conn.cursor()
|
|
|
|
print(f"Targeting {len(mappings)} channels for live ground-truth extraction...")
|
|
success = 0
|
|
|
|
for idx, (artist, info) in enumerate(mappings.items(), 1):
|
|
url = info.get("youtube", "").rstrip("/")
|
|
if not url:
|
|
continue
|
|
|
|
target_url = url if "/channel/" in url else url
|
|
|
|
try:
|
|
resp = requests.get(target_url, headers=HEADERS, timeout=12)
|
|
html = resp.text
|
|
|
|
# Match subscriber text in initial data payload
|
|
sub_match = re.search(r'"subscriberCountText":\{.*?"simpleText":"([^"]+)"', html)
|
|
if not sub_match:
|
|
sub_match = re.search(r'"accessibilityData":\{"label":"([^"]+subscribers?)"', html)
|
|
|
|
view_match = re.search(r'"viewCountText":\{.*?"simpleText":"([^"]+)"', html)
|
|
vid_match = re.search(r'"videoCountText":\{.*?"text":"([^"]+)"', html)
|
|
|
|
subs = parse_metric(sub_match.group(1)) if sub_match else 0
|
|
views = parse_metric(view_match.group(1)) if view_match else 0
|
|
vids = parse_metric(vid_match.group(1)) if vid_match else 0
|
|
|
|
if subs > 0:
|
|
success += 1
|
|
cur.execute(
|
|
"""
|
|
UPDATE telemetry_metrics
|
|
SET subscribers_count = ?
|
|
WHERE artist_id IN (
|
|
SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
|
|
)
|
|
""",
|
|
(subs, artist),
|
|
)
|
|
print(f"[{idx}/{len(mappings)}] {artist}: {subs:,} subs | {views:,} views")
|
|
else:
|
|
if idx % 15 == 0:
|
|
print(f"[{idx}/{len(mappings)}] Processing {artist}...")
|
|
except Exception:
|
|
pass
|
|
|
|
time.sleep(0.4)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"\nLive scrape complete. Updated {success} / {len(mappings)} artists with real YouTube metrics.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |