feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort
This commit is contained in:
Srilekha committed
1 parent
85e6395770
commit
78a46a8dbe
629 files changed
+123965
-755
No files matched your search
@@ -0,0 +1,250 @@
|
||||
"""
|
||||
scripts/resolve_youtube_verified_oac.py
|
||||
Resolves the remaining unresolved YouTube channels using a curated canonical
|
||||
handle dictionary and enforces a strict database safety floor.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
import requests
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parent.parent
|
||||
if str(ROOT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT_DIR))
|
||||
|
||||
from core.database import get_connection
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
# Curated handles for artists where guessing @artistname fails
|
||||
CANONICAL_HANDLES = {
|
||||
"Pusha T": "kingpush",
|
||||
"Flatbush Zombies": "FlatbushZombiesTV",
|
||||
"Larry June": "LarryJuneTFM",
|
||||
"Curren$y": "CurrensyJetLife",
|
||||
"Jeezy": "JeezyOfficial",
|
||||
"Smoke DZA": "SmokeDZA",
|
||||
"Jay-Z": "JayZVEVO",
|
||||
"Juvenile": "JuvenileVEVO",
|
||||
"Bun B": "BunBVEVO",
|
||||
"Young Buck": "YoungBuckVEVO",
|
||||
"Twista": "TwistaVEVO",
|
||||
"Cam'ron": "CamronVEVO",
|
||||
"Styles P": "StylesPVEVO",
|
||||
"Chingy": "ChingyVEVO",
|
||||
"Kool G Rap": "KoolGRapOfficial",
|
||||
"KRS-One": "KRSOneOfficial",
|
||||
"The Roots": "TheRootsVEVO",
|
||||
"Inspectah Deck": "InspectahDeckVEVO",
|
||||
"Juicy J": "JuicyJVEVO",
|
||||
"Schoolboy Q": "SchoolboyQVEVO",
|
||||
"Ab-Soul": "TopDawgEnt",
|
||||
"Quavo": "QuavoOfficial",
|
||||
"Slim Jxmmi": "RaeSremmurd",
|
||||
"Rich Homie Quan": "RichHomieQuanVEVO",
|
||||
"Key Glock": "KeyGlockVEVO",
|
||||
"Kevin Gates": "kevingatesTV",
|
||||
"Boosie Badazz": "BoosieBadazzVEVO",
|
||||
"King Von": "KingVon",
|
||||
"Trippie Redd": "TrippieRedd",
|
||||
"Ski Mask the Slump God": "SkiMaskTheSlumpGodVEVO",
|
||||
"Cordae": "cordae",
|
||||
"Mick Jenkins": "MickJenkinsVEVO",
|
||||
"Benny The Butcher": "BennyTheButcherVEVO",
|
||||
"Boldy James": "BoldyJamesVEVO",
|
||||
"Roc Marciano": "RocMarciano",
|
||||
"Maxo Kream": "MaxoKreamVEVO",
|
||||
"Big K.R.I.T.": "BigKRITVEVO",
|
||||
"Jay Worthy": "JayWorthyVEVO",
|
||||
"Veeze": "VeezeOfficial",
|
||||
"YN Jay": "YNJay",
|
||||
"Fredo Santana": "FredoSantana300",
|
||||
"Jay Critch": "JayCritchVEVO",
|
||||
"Saweetie": "OfficialSaweetie",
|
||||
"Yung Miami": "CityGirlsVEVO",
|
||||
"Tierra Whack": "TierraWhack",
|
||||
"Dizzee Rascal": "DizzeeRascalVEVO",
|
||||
"Digga D": "DiggaD",
|
||||
"K-Trap": "KTrapOfficial",
|
||||
"M Huncho": "MHunchoOfficial",
|
||||
"Nines": "NinesVEVO",
|
||||
"Potter Payper": "PotterPayperOfficial",
|
||||
"Rimzee": "RimzeeOfficial",
|
||||
"MoStack": "MoStackVEVO",
|
||||
"Stefflon Don": "StefflonDonVEVO",
|
||||
"Summrs": "SummrsArchive",
|
||||
"BabyTron": "BabyTronOfficial",
|
||||
"Luh Tyler": "LuhTylerVEVO",
|
||||
"Rob49": "Rob49VEVO",
|
||||
"Hit-Boy": "HitBoyVEVO",
|
||||
"Lil Skies": "LilSkies",
|
||||
"Smokepurpp": "SmokepurppVEVO",
|
||||
"Lil Pump": "LilPump",
|
||||
"Chris Travis": "ChrisTravisVEVO",
|
||||
"Yung Lean": "YungLeanVEVO",
|
||||
"Bladee": "DrainGang",
|
||||
"Ecco2k": "DrainGang",
|
||||
"Thaiboy Digital": "DrainGang",
|
||||
"BBNO$": "bbnomula",
|
||||
"Jhené Aiko": "JheneAiko",
|
||||
"Tinashe": "TinasheOfficial",
|
||||
"Brent Faiyaz": "BrentFaiyaz",
|
||||
"Lucky Daye": "LuckyDayeVEVO",
|
||||
"Meechy Darko": "MeechyDarkoVEVO",
|
||||
"NoCap": "NoCap16_29",
|
||||
"Rylo Rodriguez": "RyloRodriguezVEVO",
|
||||
"Lil Baby": "LilBabyVEVO",
|
||||
"EARTHGANG": "EarthgangVEVO",
|
||||
"Mobb Deep": "MobbDeepVEVO",
|
||||
"David Banner": "DavidBannerVEVO",
|
||||
"Lil Keke": "LilKekeVEVO",
|
||||
"Reason": "TopDawgEnt",
|
||||
"Foolio": "JulioFoolio",
|
||||
"Chika": "ChikaOfficial",
|
||||
"Fredo": "FredoOfficial",
|
||||
"AG Club": "AGClubOfficial",
|
||||
"Pouya": "PouyaKevin",
|
||||
"BONES": "TeamSESH",
|
||||
"Erick the Architect": "ErickTheArchitect",
|
||||
"Shy Glizzy": "ShyGlizzyVEVO",
|
||||
"Autumn!": "Twinuzis",
|
||||
"SpaceGhostPurrp": "SpaceGhostPurrp",
|
||||
"Mos Def": "MosDefVEVO",
|
||||
"SiR": "TopDawgEnt",
|
||||
"Homixide Gang": "HomixideGang",
|
||||
"BabySmoove": "BabySmooveVEVO",
|
||||
}
|
||||
|
||||
|
||||
def parse_metric(raw_text):
|
||||
if not raw_text:
|
||||
return 0
|
||||
t = str(raw_text).upper().replace(",", "").strip()
|
||||
match = re.search(r"([\d\.]+)\s*([KMB])?", t)
|
||||
if not match:
|
||||
return 0
|
||||
val = float(match.group(1))
|
||||
suffix = match.group(2)
|
||||
if suffix == "K":
|
||||
val *= 1e3
|
||||
elif suffix == "M":
|
||||
val *= 1e6
|
||||
elif suffix == "B":
|
||||
val *= 1e9
|
||||
return int(val)
|
||||
|
||||
|
||||
def extract_subscribers_from_html(html_text):
|
||||
match = re.search(r"var ytInitialData\s*=\s*({.+?});</script>", html_text)
|
||||
if not match:
|
||||
match = re.search(r"window\[\"ytInitialData\"\]\s*=\s*({.+?});", html_text)
|
||||
|
||||
if match:
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
header = data.get("header", {})
|
||||
c4 = header.get("c4TabbedHeaderRenderer", {})
|
||||
sub_text = c4.get("subscriberCountText", {}).get("simpleText", "")
|
||||
if not sub_text:
|
||||
runs = c4.get("subscriberCountText", {}).get("runs", [])
|
||||
sub_text = "".join(r.get("text", "") for r in runs)
|
||||
if sub_text:
|
||||
return parse_metric(sub_text)
|
||||
|
||||
vm = (
|
||||
header.get("pageHeaderRenderer", {})
|
||||
.get("content", {})
|
||||
.get("pageHeaderViewModel", {})
|
||||
.get("metadata", {})
|
||||
.get("contentMetadataViewModel", {})
|
||||
.get("metadataRows", [])
|
||||
)
|
||||
for row in vm:
|
||||
for part in row.get("metadataParts", []):
|
||||
text = part.get("text", {}).get("content", "")
|
||||
if "subscribers" in text.lower():
|
||||
return parse_metric(text)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
meta_match = re.search(r'([\d\.]+[KMB]?)\s+subscribers', html_text, re.IGNORECASE)
|
||||
if meta_match:
|
||||
return parse_metric(meta_match.group(1))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def fetch_handle_subs(handle):
|
||||
url = f"https://www.youtube.com/@{handle}"
|
||||
try:
|
||||
r = requests.get(url, headers=HEADERS, timeout=10)
|
||||
if r.status_code == 200:
|
||||
return extract_subscribers_from_html(r.text)
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 65)
|
||||
print("HIFFI INTELLIGENCE: CANONICAL YOUTUBE RESOLUTION & SAFETY AUDIT")
|
||||
print("=" * 65)
|
||||
|
||||
conn = get_connection()
|
||||
cur = conn.cursor()
|
||||
|
||||
updated = 0
|
||||
preserved = 0
|
||||
|
||||
for artist, handle in CANONICAL_HANDLES.items():
|
||||
# Get existing count
|
||||
cur.execute("""
|
||||
SELECT t.subscribers_count
|
||||
FROM telemetry_metrics t
|
||||
JOIN artists a ON t.artist_id = a.artist_id
|
||||
WHERE LOWER(a.artist_name) = LOWER(?)
|
||||
""", (artist,))
|
||||
row = cur.fetchone()
|
||||
existing_subs = row[0] if row and row[0] else 0
|
||||
|
||||
scraped_subs = fetch_handle_subs(handle)
|
||||
|
||||
# Enforce safety floor
|
||||
if scraped_subs > 0:
|
||||
if scraped_subs >= existing_subs or existing_subs in [0, 500000]:
|
||||
cur.execute("""
|
||||
UPDATE telemetry_metrics
|
||||
SET subscribers_count = ?
|
||||
WHERE artist_id IN (
|
||||
SELECT artist_id FROM artists WHERE LOWER(artist_name) = LOWER(?)
|
||||
)
|
||||
""", (scraped_subs, artist))
|
||||
print(f"[UPDATED] {artist:<22} -> {scraped_subs:,} subs (via @{handle})")
|
||||
updated += 1
|
||||
else:
|
||||
print(f"[PRESERVED] {artist:<22} -> Kept {existing_subs:,} subs (Scraped @{handle} was {scraped_subs:,})")
|
||||
preserved += 1
|
||||
else:
|
||||
print(f"[FAILED] {artist:<22} -> Handle @{handle} returned 0 / 404")
|
||||
|
||||
time.sleep(0.4)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print("=" * 65)
|
||||
print(f"Complete: Updated {updated} channels | Preserved {preserved} verified baselines.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user