srilekha/hiffi-artist-intelligence
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate daily velocity ingestion, circuit breaker, clean requirements, and resilient batch loader

Browse Source
This commit is contained in:
Srilekha committed 2026-10-24 11:20:10 +05:30
1 parent 66db687a58
commit 90b4956635
6 files changed
+832 -948
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pipeline/batch_500_runner.py
+130 -193
View File
@@ -1,234 +1,171 @@
"""
pipeline/batch_500_runner.py
Production Multi-Platform HIFFI Batch Runner (Hardened 100% Reliability Standards).
Fixes applied:
1. Low-follower quarantine: Scraped IG < 10,000 without verified override is treated as unverified (NaN),
preventing secondary/fan handles from dragging down veteran artists.
2. Neutral weight scaling: Prevents artists with missing signals from leapfrogging artists with verified data.
3. Strict zero-imputation & provenance tracking preserved.
HIFFI Multi-Platform Batch Runner (500-Artist Cohort).
Integrates YouTube baseline, Kworb Spotify streams & velocity (+/-), and Instagram metrics.
Enforces Bayesian neutral-anchor dual-track scoring and zero-imputation governance.
"""
import json
import sys
import os
import time
import random
from pathlib import Path
from datetime import datetime, timezone
import numpy as np
import pandas as pd
PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
import pandas as pd
import numpy as np
from collectors.wikidata_resolver import search_wikidata_qid, get_identifiers_for_qid
from collectors.browser_collector import MultiPlatformSignalCollector
from collectors.kworb_spotify_collector import KworbStreamingCollector
from playwright.sync_api import sync_playwright
from core.health_detector import ScraperCircuitBreaker
CHECKPOINT_DIR = PROJECT_ROOT / "data" / "checkpoints"
CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
CHECKPOINT_FILE = CHECKPOINT_DIR / "multiplatform_checkpoints.csv"
FINAL_OUTPUT_FILE = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"
# Candidate paths for base dataset in priority order
CANDIDATE_PATHS = [
PROJECT_ROOT / "data" / "hiffi_cohort_500.csv",
PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv",
PROJECT_ROOT / "data" / "hiphop_artists_500.csv"
]
CHECKPOINTS_PATH = PROJECT_ROOT / "data" / "checkpoints" / "multiplatform_checkpoints.csv"
OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json"
CSV_EXPORT_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"
def min_max_normalize(series: pd.Series) -> pd.Series:
"""Scales numeric series strictly between 0 and 100 while preserving NaNs."""
def get_base_dataset_path() -> Path:
"""Finds the existing base dataset across standard project names."""
for path in CANDIDATE_PATHS:
if path.exists():
return path
raise FileNotFoundError(
f"Base dataset not found. Checked: {[str(p) for p in CANDIDATE_PATHS]}"
)
def min_max_norm(series: pd.Series, neutral_val: float) -> pd.Series:
"""Normalizes a metric to a 0-100 scale, falling back to a neutral value for NaNs."""
valid = series.dropna()
if valid.empty or valid.max() == valid.min():
return pd.Series(np.nan, index=series.index)
return ((series - valid.min()) / (valid.max() - valid.min())) * 100.0
return pd.Series(neutral_val, index=series.index)
norm = (series - valid.min()) / (valid.max() - valid.min()) * 100.0
return norm.fillna(neutral_val)
class BatchMultiPlatformRunner:
def __init__(self, youtube_csv_path: str = "enriched_artists.csv"):
self.input_file = PROJECT_ROOT / youtube_csv_path
self.df_base = pd.read_csv(self.input_file)
self.df_base["name_clean"] = self.df_base["name"].astype(str).str.strip().str.lower()
self.collector = MultiPlatformSignalCollector(headless=True)
self.kworb_collector = KworbStreamingCollector()
def main():
print("=" * 90)
print("HIFFI MULTI-PLATFORM RUNNER (100% RELIABILITY & VERIFIED GOVERNANCE)")
print("Total Base Cohort: 500 artists")
print("=" * 90)
def load_checkpoint(self) -> pd.DataFrame:
if CHECKPOINT_FILE.exists():
return pd.read_csv(CHECKPOINT_FILE)
return pd.DataFrame(columns=[
"name", "name_clean", "wikidata_qid", "spotify_id",
"instagram_handle", "instagram_followers", "resolution_status", "ig_status"
])
# 1. Resolve and Load Base Dataset
base_path = get_base_dataset_path()
print(f"[Loader] Loading base cohort from: {base_path.name}")
df = pd.read_csv(base_path)
def save_checkpoint(self, df_checkpoint: pd.DataFrame):
df_checkpoint.to_csv(CHECKPOINT_FILE, index=False)
# 2. Merge Checkpoints and Overrides
if CHECKPOINTS_PATH.exists():
chk = pd.read_csv(CHECKPOINTS_PATH)
merge_cols = [c for c in ["name", "instagram_followers", "instagram_handle"] if c in chk.columns]
df = df.drop(columns=[c for c in ["instagram_followers", "instagram_handle"] if c in df.columns], errors="ignore")
df = pd.merge(df, chk[merge_cols], on="name", how="left")
def run_full_pipeline(self, batch_size: int = 25, delay_range: tuple = (0.5, 1.2)):
print("=" * 90)
print("HIFFI MULTI-PLATFORM RUNNER (100% RELIABILITY & VERIFIED GOVERNANCE)")
print(f"Total Base Cohort: {len(self.df_base)} artists")
print("=" * 90)
if OVERRIDES_PATH.exists():
with open(OVERRIDES_PATH, "r") as f:
overrides = json.load(f)
for artist, data in overrides.items():
mask = df["name"].str.strip().str.lower() == artist.strip().lower()
if mask.any():
if "instagram_handle" in data:
df.loc[mask, "instagram_handle"] = data["instagram_handle"]
if "instagram_followers" in data:
df.loc[mask, "instagram_followers"] = data["instagram_followers"]
df_ckpt = self.load_checkpoint()
processed_names = set(df_ckpt["name_clean"].dropna().tolist())
# Quarantine suspicious handles with < 10,000 followers to prevent dormant penalties
df.loc[df["instagram_followers"] < 10000, "instagram_followers"] = np.nan
artists_to_process = self.df_base[~self.df_base["name_clean"].isin(processed_names)].copy()
if not artists_to_process.empty:
print(f"Artists Remaining to Process: {len(artists_to_process)}")
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page(
viewport={"width": 1920, "height": 1080},
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
)
# 3. Ingest Kworb Public Streaming Metrics & Daily Velocity
print("\n" + "=" * 90)
print("PHASE 2: MERGING SIGNALS & ENFORCING RELIABILITY CRITERIA")
print("=" * 90)
new_rows = []
for idx, row in artists_to_process.iterrows():
artist_name = str(row["name"]).strip()
name_clean = artist_name.lower()
current_count = len(df_ckpt) + len(new_rows) + 1
kworb = KworbStreamingCollector()
kworb.load_aggregator_tables(max_pages=2)
qid = search_wikidata_qid(artist_name)
ids = get_identifiers_for_qid(qid) if qid else {"spotify_id": "N/A", "instagram_handle": "N/A"}
handle = ids.get("instagram_handle", "N/A")
ig_followers = None
ig_status = "NO_HANDLE"
df["spotify_monthly_listeners"] = df["name"].apply(lambda n: kworb.get_monthly_listeners(n))
df["spotify_daily_change"] = df["name"].apply(lambda n: kworb.get_daily_change(n))
if handle and handle != "N/A":
ig_url = f"https://www.instagram.com/{handle}/"
res = self.collector.collect_artist({"name": artist_name, "ig_url": ig_url}, page)
ig_followers = res.get("instagram_followers")
ig_status = res.get("ig_status", "FAILED")
# Audit Provenance Tracking
df["youtube_data_source"] = "youtube_data_api_v3"
df["spotify_data_source"] = df["spotify_monthly_listeners"].apply(
lambda x: "kworb_public_aggregator" if pd.notna(x) else None
)
df["spotify_status"] = df["spotify_monthly_listeners"].apply(
lambda x: "VERIFIED_PUBLIC_AGGREGATOR" if pd.notna(x) else "UNAVAILABLE_IN_TOP_5000"
)
df["instagram_data_source"] = df["instagram_followers"].apply(
lambda x: "playwright_meta_scrape" if pd.notna(x) else None
)
new_rows.append({
"name": artist_name,
"name_clean": name_clean,
"wikidata_qid": qid or "N/A",
"spotify_id": ids.get("spotify_id", "N/A"),
"instagram_handle": handle,
"instagram_followers": ig_followers,
"resolution_status": "RESOLVED" if qid else "NOT_FOUND",
"ig_status": ig_status
})
# Completeness Flag
df["has_complete_multiplatform_signals"] = (
df["production_hiffi_score"].notna() &
df["spotify_monthly_listeners"].notna() &
df["instagram_followers"].notna()
)
if len(new_rows) % batch_size == 0:
df_ckpt = pd.concat([df_ckpt, pd.DataFrame(new_rows)], ignore_index=True)
self.save_checkpoint(df_ckpt)
new_rows = []
# -------------------------------------------------------------
# DUAL-TRACK SCORING WITH VELOCITY ENHANCEMENT
# -------------------------------------------------------------
# Production baseline is strictly preserved
if "production_hiffi_score" not in df.columns:
df["production_hiffi_score"] = 50.0
if "production_hiffi_rank" not in df.columns:
df["production_hiffi_rank"] = df["production_hiffi_score"].rank(ascending=False, method="min").astype(int)
time.sleep(random.uniform(*delay_range))
# Normalized Component Scores with Bayesian Neutral Anchors
yt_norm = min_max_norm(df["production_hiffi_score"], neutral_val=50.0)
if new_rows:
df_ckpt = pd.concat([df_ckpt, pd.DataFrame(new_rows)], ignore_index=True)
self.save_checkpoint(df_ckpt)
spotify_valid = df["spotify_monthly_listeners"].dropna()
spotify_median = (spotify_valid.median() - spotify_valid.min()) / (spotify_valid.max() - spotify_valid.min()) * 100.0 if not spotify_valid.empty else 50.0
spotify_norm = min_max_norm(df["spotify_monthly_listeners"], neutral_val=spotify_median)
browser.close()
ig_valid = df["instagram_followers"].dropna()
ig_median = (ig_valid.median() - ig_valid.min()) / (ig_valid.max() - ig_valid.min()) * 100.0 if not ig_valid.empty else 50.0
ig_norm = min_max_norm(df["instagram_followers"], neutral_val=ig_median)
print("\n" + "=" * 90)
print("PHASE 2: MERGING SIGNALS & ENFORCING RELIABILITY CRITERIA")
print("=" * 90)
# Daily Velocity Calculation: Ratio of Daily Change to Total Listeners
# Missing velocity data defaults to a neutral 0.0 momentum adjustment (no synthetic penalties)
velocity_ratio = df["spotify_daily_change"] / df["spotify_monthly_listeners"]
valid_velocity = velocity_ratio.dropna()
merged = pd.merge(
self.df_base,
df_ckpt,
on="name_clean",
how="left",
suffixes=("", "_ext")
)
if not valid_velocity.empty and valid_velocity.std() > 0:
norm_velocity = (velocity_ratio - valid_velocity.mean()) / valid_velocity.std()
momentum_adjustment = np.clip(norm_velocity.fillna(0.0) * 1.5, -5.0, 5.0)
else:
momentum_adjustment = 0.0
now_utc = datetime.now(timezone.utc).isoformat()
# Composite HIFFI-XP Score (Bounded strictly between 0.0 and 100.0)
raw_xp = (0.45 * yt_norm) + (0.35 * spotify_norm) + (0.20 * ig_norm) + momentum_adjustment
df["hiffi_xp_score"] = np.clip(raw_xp, 0.0, 100.0).round(2)
df["hiffi_xp_rank"] = df["hiffi_xp_score"].rank(ascending=False, method="min").astype(int)
# Ingest Verified Streaming Telemetry from Kworb Aggregator
self.kworb_collector.load_aggregator_tables(max_pages=2)
spotify_listeners = []
spotify_statuses = []
spotify_sources = []
spotify_collected_ats = []
# 4. Export Final Audited Dataset
CSV_EXPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(CSV_EXPORT_PATH, index=False)
print(f"\nFinal Hardened Multi-Platform Dataset Exported: {CSV_EXPORT_PATH}")
for _, row in merged.iterrows():
artist_name = row["name"]
listeners = self.kworb_collector.get_monthly_listeners(artist_name)
if listeners is not None:
spotify_listeners.append(listeners)
spotify_statuses.append("VERIFIED_PUBLIC_AGGREGATOR")
spotify_sources.append("kworb_public_aggregator")
spotify_collected_ats.append(now_utc)
else:
spotify_listeners.append(np.nan)
spotify_statuses.append("UNAVAILABLE_IN_TOP_5000")
spotify_sources.append(None)
spotify_collected_ats.append(None)
# 5. Pipeline Data Reliability & Audit Summary
verified_spotify = df["spotify_monthly_listeners"].notna().sum()
verified_ig = (df["instagram_followers"] >= 10000).sum()
complete_signals = df["has_complete_multiplatform_signals"].sum()
merged["spotify_monthly_listeners"] = spotify_listeners
merged["spotify_popularity"] = np.nan
merged["spotify_status"] = spotify_statuses
merged["spotify_data_source"] = spotify_sources
merged["spotify_collected_at"] = spotify_collected_ats
merged["is_spotify_verified"] = merged["spotify_monthly_listeners"].notna()
# Instagram Telemetry Sanitization:
# Strict reliability check: followers < 10,000 for major artists without verified status are marked NaN
raw_followers = pd.to_numeric(merged["instagram_followers"], errors="coerce")
sanitized_followers = []
for f in raw_followers:
if pd.isna(f) or f < 10000:
sanitized_followers.append(np.nan)
else:
sanitized_followers.append(f)
merged["instagram_followers"] = sanitized_followers
merged["instagram_collected_at"] = np.where(merged["instagram_followers"].notna(), now_utc, None)
merged["instagram_data_source"] = np.where(merged["instagram_followers"].notna(), "playwright_meta_scrape", None)
merged["is_instagram_handle_verified"] = merged["instagram_followers"].notna()
# YouTube Production Telemetry
merged["youtube_data_source"] = "youtube_data_api_v3"
merged["yt_baseline_score"] = pd.to_numeric(merged["hiffi_score"], errors="coerce")
# Official Production Rank (YouTube Ground Truth Baseline)
merged["production_hiffi_score"] = merged["yt_baseline_score"]
merged["production_hiffi_rank"] = merged["production_hiffi_score"].rank(ascending=False, method="min")
# Multi-Platform Normalization
norm_yt = min_max_normalize(merged["yt_baseline_score"]).fillna(0)
norm_sp = min_max_normalize(merged["spotify_monthly_listeners"])
norm_ig = min_max_normalize(merged["instagram_followers"])
# Cohort Medians used strictly as neutral anchors (Prevents NaN leapfrogging)
median_sp = norm_sp.dropna().median()
median_ig = norm_ig.dropna().median()
xp_scores = []
for i in range(len(merged)):
yt_val = norm_yt.iloc[i]
sp_val = norm_sp.iloc[i] if pd.notna(norm_sp.iloc[i]) else median_sp
ig_val = norm_ig.iloc[i] if pd.notna(norm_ig.iloc[i]) else median_ig
# Weighted formula: 45% YouTube, 35% Spotify, 20% Instagram
score = (0.45 * yt_val) + (0.35 * sp_val) + (0.20 * ig_val)
xp_scores.append(round(score, 2))
merged["hiffi_xp_score"] = xp_scores
merged["hiffi_xp_rank"] = merged["hiffi_xp_score"].rank(ascending=False, method="min").astype(int)
merged["has_complete_multiplatform_signals"] = (
merged["spotify_monthly_listeners"].notna() &
merged["instagram_followers"].notna() &
merged["yt_baseline_score"].notna()
)
FINAL_OUTPUT_FILE.parent.mkdir(parents=True, exist_ok=True)
merged.to_csv(FINAL_OUTPUT_FILE, index=False)
print(f"\nFinal Hardened Multi-Platform Dataset Exported: {FINAL_OUTPUT_FILE}")
print("\n" + "=" * 90)
print("PIPELINE DATA RELIABILITY & AUDIT SUMMARY")
print("=" * 90)
print(f"Total Cohort Rows: {len(merged)}")
print(f"Spotify Verified Telemetry Ingest: {merged['spotify_monthly_listeners'].notna().sum()} (Source: kworb_public_aggregator)")
print(f"Verified Instagram Reach (>10k): {merged['instagram_followers'].notna().sum()} / {len(merged)}")
print(f"Complete 3-Platform Signals: {merged['has_complete_multiplatform_signals'].sum()} / {len(merged)}")
print("-" * 90)
print("\n" + "=" * 90)
print("PIPELINE DATA RELIABILITY & AUDIT SUMMARY")
print("=" * 90)
print(f"Total Cohort Rows: {len(df)}")
print(f"Spotify Verified Telemetry Ingest: {verified_spotify} (Source: kworb_public_aggregator)")
print(f"Verified Instagram Reach (>10k): {verified_ig} / {len(df)}")
print(f"Complete 3-Platform Signals: {complete_signals} / {len(df)}")
print("-" * 90)
if __name__ == "__main__":
runner = BatchMultiPlatformRunner("enriched_artists.csv")
runner.run_full_pipeline(batch_size=15, delay_range=(0.5, 1.0))
main()