250 lines
7.7 KiB
Python
250 lines
7.7 KiB
Python
|
"""
|
||
|
|
scripts/resolve_youtube_verified_oac.py
|
||
|
|
Resolves the remaining unresolved YouTube channels using a curated canonical
|
||
|
|
handle dictionary and enforces a strict database safety floor.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from pathlib import Path
|
||
|
|
import requests
|
||
|
|
|
||
|
|
ROOT_DIR = Path(__file__).resolve().parent.parent
|
||
|
|
if str(ROOT_DIR) not in sys.path:
|
||
|
|
sys.path.insert(0, str(ROOT_DIR))
|
||
|
|
|
||
|
|
from core.database import get_connection
|
||
|
|
|
||
|
|
HEADERS = {
|
||
|
|
"User-Agent": (
|
||
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
||
|
|
),
|
||
|
|
"Accept-Language": "en-US,en;q=0.9",
|
||
|
|
}
|
||
|
|
|
||
|
|
# Curated handles for artists where guessing @artistname fails
|
||
|
|
CANONICAL_HANDLES = {
|
||
|
|
"Pusha T": "kingpush",
|
||
|
|
"Flatbush Zombies": "FlatbushZombiesTV",
|
||
|
|
"Larry June": "LarryJuneTFM",
|
||
|
|
"Curren$y": "CurrensyJetLife",
|
||
|
|
"Jeezy": "JeezyOfficial",
|
||
|
|
"Smoke DZA": "SmokeDZA",
|
||
|
|
"Jay-Z": "JayZVEVO",
|
||
|
|
"Juvenile": "JuvenileVEVO",
|
||
|
|
"Bun B": "BunBVEVO",
|
||
|
|
"Young Buck": "YoungBuckVEVO",
|
||
|
|
"Twista": "TwistaVEVO",
|
||
|
|
"Cam'ron": "CamronVEVO",
|
||
|
|
"Styles P": "StylesPVEVO",
|
||
|
|
"Chingy": "ChingyVEVO",
|
||
|
|
"Kool G Rap": "KoolGRapOfficial",
|
||
|
|
"KRS-One": "KRSOneOfficial",
|
||
|
|
"The Roots": "TheRootsVEVO",
|
||
|
|
"Inspectah Deck": "InspectahDeckVEVO",
|
||
|
|
"Juicy J": "JuicyJVEVO",
|
||
|
|
"Schoolboy Q": "SchoolboyQVEVO",
|
||
|
|
"Ab-Soul": "TopDawgEnt",
|
||
|
|
"Quavo": "QuavoOfficial",
|
||
|
|
"Slim Jxmmi": "RaeSremmurd",
|
||
|
|
"Rich Homie Quan": "RichHomieQuanVEVO",
|
||
|
|
"Key Glock": "KeyGlockVEVO",
|
||
|
|
"Kevin Gates": "kevingatesTV",
|
||
|
|
"Boosie Badazz": "BoosieBadazzVEVO",
|
||
|
|
"King Von": "KingVon",
|
||
|
|
"Trippie Redd": "TrippieRedd",
|
||
|
|
"Ski Mask the Slump God": "SkiMaskTheSlumpGodVEVO",
|
||
|
|
"Cordae": "cordae",
|
||
|
|
"Mick Jenkins": "MickJenkinsVEVO",
|
||
|
|
"Benny The Butcher": "BennyTheButcherVEVO",
|
||
|
|
"Boldy James": "BoldyJamesVEVO",
|
||
|
|
"Roc Marciano": "RocMarciano",
|
||
|
|
"Maxo Kream": "MaxoKreamVEVO",
|
||
|
|
"Big K.R.I.T.": "BigKRITVEVO",
|
||
|
|
"Jay Worthy": "JayWorthyVEVO",
|
||
|
|
"Veeze": "VeezeOfficial",
|
||
|
|
"YN Jay": "YNJay",
|
||
|
|
"Fredo Santana": "FredoSantana300",
|
||
|
|
"Jay Critch": "JayCritchVEVO",
|
||
|
|
"Saweetie": "OfficialSaweetie",
|
||
|
|
"Yung Miami": "CityGirlsVEVO",
|
||
|
|
"Tierra Whack": "TierraWhack",
|
||
|
|
"Dizzee Rascal": "DizzeeRascalVEVO",
|
||
|
|
"Digga D": "DiggaD",
|
||
|
|
"K-Trap": "KTrapOfficial",
|
||
|
|
"M Huncho": "MHunchoOfficial",
|
||
|
|
"Nines": "NinesVEVO",
|
||
|
|
"Potter Payper": "PotterPayperOfficial",
|
||
|
|
"Rimzee": "RimzeeOfficial",
|
||
|
|
"MoStack": "MoStackVEVO",
|
||
|
|
"Stefflon Don": "StefflonDonVEVO",
|
||
|
|
"Summrs": "SummrsArchive",
|
||
|
|
"BabyTron": "BabyTronOfficial",
|
||
|
|
"Luh Tyler": "LuhTylerVEVO",
|
||
|
|
"Rob49": "Rob49VEVO",
|
||
|
|
"Hit-Boy": "HitBoyVEVO",
|
||
|
|
"Lil Skies": "LilSkies",
|
||
|
|
"Smokepurpp": "SmokepurppVEVO",
|
||
|
|
"Lil Pump": "LilPump",
|
||
|
|
"Chris Travis": "ChrisTravisVEVO",
|
||
|
|
"Yung Lean": "YungLeanVEVO",
|
||
|
|
"Bladee": "DrainGang",
|
||
|
|
"Ecco2k": "DrainGang",
|
||
|
|
"Thaiboy Digital": "DrainGang",
|
||
|
|
"BBNO$": "bbnomula",
|
||
|
|
"Jhené Aiko": "JheneAiko",
|
||
|
|
"Tinashe": "TinasheOfficial",
|
||
|
|
"Brent Faiyaz": "BrentFaiyaz",
|
||
|
|
"Lucky Daye": "LuckyDayeVEVO",
|
||
|
|
"Meechy Darko": "MeechyDarkoVEVO",
|
||
|
|
"NoCap": "NoCap16_29",
|
||
|
|
"Rylo Rodriguez": "RyloRodriguezVEVO",
|
||
|
|
"Lil Baby": "LilBabyVEVO",
|
||
|
|
"EARTHGANG": "EarthgangVEVO",
|
||
|
|
"Mobb Deep": "MobbDeepVEVO",
|
||
|
|
"David Banner": "DavidBannerVEVO",
|
||
|
|
"Lil Keke": "LilKekeVEVO",
|
||
|
|
"Reason": "TopDawgEnt",
|
||
|
|
"Foolio": "JulioFoolio",
|
||
|
|
"Chika": "ChikaOfficial",
|
||
|
|
"Fredo": "FredoOfficial",
|
||
|
|
"AG Club": "AGClubOfficial",
|
||
|
|
"Pouya": "PouyaKevin",
|
||
|
|
"BONES": "TeamSESH",
|
||
|
|
"Erick the Architect": "ErickTheArchitect",
|
||
|
|
"Shy Glizzy": "ShyGlizzyVEVO",
|
||
|
|
"Autumn!": "Twinuzis",
|
||
|
|
"SpaceGhostPurrp": "SpaceGhostPurrp",
|
||
|
|
"Mos Def": "MosDefVEVO",
|
||
|
|
"SiR": "TopDawgEnt",
|
||
|
|
"Homixide Gang": "HomixideGang",
|
||
|
|
"BabySmoove": "BabySmooveVEVO",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def parse_metric(raw_text):
|
||
|
|
if not raw_text:
|
||
|
|
return 0
|
||
|
|
t = str(raw_text).upper().replace(",", "").strip()
|
||
|
|
match = re.search(r"([\d\.]+)\s*([KMB])?", t)
|
||
|
|
if not match:
|
||
|
|
return 0
|
||
|
|
val = float(match.group(1))
|
||
|
|
suffix = match.group(2)
|
||
|
|
if suffix == "K":
|
||
|
|
val *= 1e3
|
||
|
|
elif suffix == "M":
|
||
|
|
val *= 1e6
|
||
|
|
elif suffix == "B":
|
||
|
|
val *= 1e9
|
||
|
|
return int(val)
|
||
|
|
|
||
|
|
|
||
|
|
def extract_subscribers_from_html(html_text):
|
||
|
|
match = re.search(r"var ytInitialData\s*=\s*({.+?});</script>", html_text)
|
||
|
|
if not match:
|
||
|
|
match = re.search(r"window\[\"ytInitialData\"\]\s*=\s*({.+?});", html_text)
|
||
|
|
|
||
|
|
if match:
|
||
|
|
try:
|
||
|
|
data = json.loads(match.group(1))
|
||
|
|
header = data.get("header", {})
|
||
|
|
c4 = header.get("c4TabbedHeaderRenderer", {})
|
||
|
|
sub_text = c4.get("subscriberCountText", {}).get("simpleText", "")
|
||
|
|
if not sub_text:
|
||
|
|
runs = c4.get("subscriberCountText", {}).get("runs", [])
|
||
|
|
sub_text = "".join(r.get("text", "") for r in runs)
|
||
|
|
if sub_text:
|
||
|
|
return parse_metric(sub_text)
|
||
|
|
|
||
|
|
vm = (
|
||
|
|
header.get("pageHeaderRenderer", {})
|
||
|
|
.get("content", {})
|
||
|
|
.get("pageHeaderViewModel", {})
|
||
|
|
.get("metadata", {})
|
||
|
|
.get("contentMetadataViewModel", {})
|
||
|
|
.get("metadataRows", [])
|
||
|
|
)
|
||
|
|
for row in vm:
|
||
|
|
for part in row.get("metadataParts", []):
|
||
|
|
text = part.get("text", {}).get("content", "")
|
||
|
|
if "subscribers" in text.lower():
|
||
|
|
return parse_metric(text)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
|
||
|
|
meta_match = re.search(r'([\d\.]+[KMB]?)\s+subscribers', html_text, re.IGNORECASE)
|
||
|
|
if meta_match:
|
||
|
|
return parse_metric(meta_match.group(1))
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_handle_subs(handle):
|
||
|
|
url = f"https://www.youtube.com/@{handle}"
|
||
|
|
try:
|
||
|
|
r = requests.get(url, headers=HEADERS, timeout=10)
|
||
|
|
if r.status_code == 200:
|
||
|
|
return extract_subscribers_from_html(r.text)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
print("=" * 65)
|
||
|
|
print("HIFFI INTELLIGENCE: CANONICAL YOUTUBE RESOLUTION & SAFETY AUDIT")
|
||
|
|
print("=" * 65)
|
||
|
|
|
||
|
|
conn = get_connection()
|
||
|
|
cur = conn.cursor()
|
||
|
|
|
||
|
|
updated = 0
|
||
|
|
preserved = 0
|
||
|
|
|
||
|
|
for artist, handle in CANONICAL_HANDLES.items():
|
||
|
|
# Get existing count
|
||
|
|
cur.execute("""
|
||
|
|
SELECT t.subscribers_count
|
||
|
|
FROM telemetry_metrics t
|
||
|
|
JOIN artists a ON t.artist_id = a.artist_id
|
||
|
|
WHERE LOWER(a.artist_name) = LOWER(?)
|
||
|
|
""", (artist,))
|
||
|
|
row = cur.fetchone()
|
||
|
|
existing_subs = row[0] if row and row[0] else 0
|
||
|
|
|
||
|
|
scraped_subs = fetch_handle_subs(handle)
|
||
|
|
|
||
|
|
# Enforce safety floor
|
||
|
|
if scraped_subs > 0:
|
||
|
|
if scraped_subs >= existing_subs or existing_subs in [0, 500000]:
|
||
|
|
cur.execute("""
|
||
|
|
UPDATE telemetry_metrics
|
||
|
|
SET subscribers_count = ?
|
||
|
|
WHERE artist_id IN (
|
||
|
|
SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
|
||
|
|
)
|
||
|
|
""", (scraped_subs, artist))
|
||
|
|
print(f"[UPDATED] {artist:<22} -> {scraped_subs:,} subs (via @{handle})")
|
||
|
|
updated += 1
|
||
|
|
else:
|
||
|
|
print(f"[PRESERVED] {artist:<22} -> Kept {existing_subs:,} subs (Scraped @{handle} was {scraped_subs:,})")
|
||
|
|
preserved += 1
|
||
|
|
else:
|
||
|
|
print(f"[FAILED] {artist:<22} -> Handle @{handle} returned 0 / 404")
|
||
|
|
|
||
|
|
time.sleep(0.4)
|
||
|
|
|
||
|
|
conn.commit()
|
||
|
|
conn.close()
|
||
|
|
|
||
|
|
print("=" * 65)
|
||
|
|
print(f"Complete: Updated {updated} channels | Preserved {preserved} verified baselines.")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|