145 lines
4.2 KiB
Python
145 lines
4.2 KiB
Python
"""
|
|
scripts/resolve_and_scrape_remaining_channels.py
|
|
Automatically discovers canonical channel IDs and extracts live subscriber
|
|
and view telemetry for any cohort artist currently unverified.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
import requests
|
|
|
|
ROOT_DIR = Path(__file__).resolve().parent.parent
|
|
if str(ROOT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(ROOT_DIR))
|
|
|
|
from core.database import get_connection
|
|
|
|
HEADERS = {
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
|
),
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
}
|
|
|
|
|
|
def parse_metric(raw_text):
|
|
if not raw_text:
|
|
return 0
|
|
t = str(raw_text).upper().replace(",", "").strip()
|
|
match = re.search(r"([\d\.]+)\s*([KMB])?", t)
|
|
if not match:
|
|
return 0
|
|
val = float(match.group(1))
|
|
suffix = match.group(2)
|
|
if suffix == "K":
|
|
val *= 1e3
|
|
elif suffix == "M":
|
|
val *= 1e6
|
|
elif suffix == "B":
|
|
val *= 1e9
|
|
return int(val)
|
|
|
|
|
|
def extract_yt_initial_data(html_text):
|
|
match = re.search(r"var ytInitialData\s*=\s*({.+?});</script>", html_text)
|
|
if not match:
|
|
match = re.search(r"window\[\"ytInitialData\"\]\s*=\s*({.+?});", html_text)
|
|
if match:
|
|
try:
|
|
return json.loads(match.group(1))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return None
|
|
|
|
|
|
def search_channel(artist_name):
|
|
query = f"{artist_name} official channel"
|
|
url = f"https://www.youtube.com/results?search_query={urllib.parse.quote_plus(query)}"
|
|
try:
|
|
r = requests.get(url, headers=HEADERS, timeout=12)
|
|
if r.status_code != 200:
|
|
return None
|
|
|
|
data = extract_yt_initial_data(r.text)
|
|
if not data:
|
|
return None
|
|
|
|
# Traverse contents to find channelRenderer
|
|
contents = (
|
|
data.get("contents", {})
|
|
.get("twoColumnSearchResultsRenderer", {})
|
|
.get("primaryContents", {})
|
|
.get("sectionListRenderer", {})
|
|
.get("contents", [])
|
|
)
|
|
|
|
for sec in contents:
|
|
item_section = sec.get("itemSectionRenderer", {}).get("contents", [])
|
|
for item in item_section:
|
|
if "channelRenderer" in item:
|
|
cr = item["channelRenderer"]
|
|
sub_text = cr.get("subscriberCountText", {}).get("simpleText", "")
|
|
if not sub_text:
|
|
runs = cr.get("subscriberCountText", {}).get("runs", [])
|
|
sub_text = "".join(r.get("text", "") for r in runs)
|
|
|
|
channel_id = cr.get("channelId")
|
|
subs = parse_metric(sub_text)
|
|
if subs > 0:
|
|
return {"channel_id": channel_id, "subscribers": subs}
|
|
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
|
|
def main():
|
|
conn = get_connection()
|
|
cur = conn.cursor()
|
|
|
|
# Find cohort artists that need ground truth update
|
|
cur.execute("""
|
|
SELECT a.artist_name
|
|
FROM artists a
|
|
JOIN telemetry_metrics t ON a.artist_id = t.artist_id
|
|
WHERE t.subscribers_count = 500000 OR t.subscribers_count IS NULL
|
|
""")
|
|
targets = [row[0] for row in cur.fetchall()]
|
|
|
|
print(f"Resolving verified channels for {len(targets)} artists...")
|
|
success = 0
|
|
|
|
for idx, artist in enumerate(targets, 1):
|
|
res = search_channel(artist)
|
|
if res and res["subscribers"] > 0:
|
|
success += 1
|
|
cur.execute(
|
|
"""
|
|
UPDATE telemetry_metrics
|
|
SET subscribers_count = ?
|
|
WHERE artist_id IN (
|
|
SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
|
|
)
|
|
""",
|
|
(res["subscribers"], artist),
|
|
)
|
|
print(f"[{idx}/{len(targets)}] {artist} -> {res['subscribers']:,} subs (ID: {res['channel_id']})")
|
|
else:
|
|
if idx % 10 == 0:
|
|
print(f"[{idx}/{len(targets)}] In progress: {artist}...")
|
|
|
|
time.sleep(0.5)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"\nCompleted: Verified and updated {success} / {len(targets)} channels.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |