feat: add Instagram candidate verifier, Spotify fallback collector, and expand QA suite to 12 tests
This commit is contained in:
Srilekha committed
1 parent
90b4956635
commit
62aea7a85b
6 files changed
+1168
-651
No files matched your search
+176
-85
@@ -1,11 +1,14 @@
|
||||
"""
|
||||
pipeline/batch_500_runner.py
|
||||
HIFFI Multi-Platform Batch Runner (500-Artist Cohort).
|
||||
Integrates YouTube baseline, Kworb Spotify streams & velocity (+/-), and Instagram metrics.
|
||||
Enforces Bayesian neutral-anchor dual-track scoring and zero-imputation governance.
|
||||
Integrates YouTube baseline, Kworb Spotify telemetry, Spotify Embed Fallback,
|
||||
and Multi-Signal Instagram Candidate Verification.
|
||||
Enforces strict dual-track scoring and zero-imputation governance.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -17,27 +20,20 @@ if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from collectors.kworb_spotify_collector import KworbStreamingCollector
|
||||
from collectors.spotify_fallback_collector import SpotifyFallbackCollector
|
||||
from collectors.instagram_verifier import InstagramCandidateVerifier
|
||||
from core.health_detector import ScraperCircuitBreaker
|
||||
|
||||
# Candidate paths for base dataset in priority order
|
||||
CANDIDATE_PATHS = [
|
||||
PROJECT_ROOT / "data" / "hiffi_cohort_500.csv",
|
||||
PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv",
|
||||
PROJECT_ROOT / "data" / "hiphop_artists_500.csv"
|
||||
]
|
||||
CHECKPOINTS_PATH = PROJECT_ROOT / "data" / "checkpoints" / "multiplatform_checkpoints.csv"
|
||||
OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json"
|
||||
CSV_EXPORT_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"
|
||||
|
||||
|
||||
def get_base_dataset_path() -> Path:
|
||||
"""Finds the existing base dataset across standard project names."""
|
||||
for path in CANDIDATE_PATHS:
|
||||
if path.exists():
|
||||
return path
|
||||
raise FileNotFoundError(
|
||||
f"Base dataset not found. Checked: {[str(p) for p in CANDIDATE_PATHS]}"
|
||||
)
|
||||
def clean_norm_key(text: object) -> str:
|
||||
"""Strips non-alphanumeric characters for fuzzy-safe key matching."""
|
||||
if text is None or pd.isna(text):
|
||||
return ""
|
||||
return re.sub(r"[^a-zA-Z0-9]", "", str(text)).lower()
|
||||
|
||||
|
||||
def min_max_norm(series: pd.Series, neutral_val: float) -> pd.Series:
|
||||
@@ -51,120 +47,215 @@ def min_max_norm(series: pd.Series, neutral_val: float) -> pd.Series:
|
||||
|
||||
def main():
|
||||
print("=" * 90)
|
||||
print("HIFFI MULTI-PLATFORM RUNNER (100% RELIABILITY & VERIFIED GOVERNANCE)")
|
||||
print("HIFFI MULTI-PLATFORM RUNNER (VERIFIED SIGNALS & STRICT GOVERNANCE)")
|
||||
print("Total Base Cohort: 500 artists")
|
||||
print("=" * 90)
|
||||
|
||||
# 1. Resolve and Load Base Dataset
|
||||
base_path = get_base_dataset_path()
|
||||
print(f"[Loader] Loading base cohort from: {base_path.name}")
|
||||
df = pd.read_csv(base_path)
|
||||
# 1. Load Base Cohort
|
||||
if not CSV_EXPORT_PATH.exists():
|
||||
raise FileNotFoundError(f"Base dataset missing at {CSV_EXPORT_PATH}")
|
||||
print(f"[Loader] Loading base cohort from: {CSV_EXPORT_PATH.name}")
|
||||
df = pd.read_csv(CSV_EXPORT_PATH)
|
||||
df = df.drop_duplicates(subset=["name"]).reset_index(drop=True)
|
||||
|
||||
# 2. Merge Checkpoints and Overrides
|
||||
# 2. Build Robust Checkpoint Index from multiplatform_checkpoints.csv (341 records)
|
||||
chk_by_name = {}
|
||||
chk_by_handle = {}
|
||||
if CHECKPOINTS_PATH.exists():
|
||||
chk = pd.read_csv(CHECKPOINTS_PATH)
|
||||
merge_cols = [c for c in ["name", "instagram_followers", "instagram_handle"] if c in chk.columns]
|
||||
df = df.drop(columns=[c for c in ["instagram_followers", "instagram_handle"] if c in df.columns], errors="ignore")
|
||||
df = pd.merge(df, chk[merge_cols], on="name", how="left")
|
||||
for _, r in chk.iterrows():
|
||||
raw_name = str(r.get("name", "")).strip()
|
||||
norm_name = clean_norm_key(raw_name)
|
||||
raw_handle = str(r.get("instagram_handle", "")).replace("@", "").strip()
|
||||
norm_handle = clean_norm_key(raw_handle)
|
||||
|
||||
if OVERRIDES_PATH.exists():
|
||||
with open(OVERRIDES_PATH, "r") as f:
|
||||
overrides = json.load(f)
|
||||
for artist, data in overrides.items():
|
||||
mask = df["name"].str.strip().str.lower() == artist.strip().lower()
|
||||
if mask.any():
|
||||
if "instagram_handle" in data:
|
||||
df.loc[mask, "instagram_handle"] = data["instagram_handle"]
|
||||
if "instagram_followers" in data:
|
||||
df.loc[mask, "instagram_followers"] = data["instagram_followers"]
|
||||
f_val = r.get("instagram_followers")
|
||||
parsed_followers = None
|
||||
if pd.notna(f_val):
|
||||
try:
|
||||
f_num = float(f_val)
|
||||
if not np.isnan(f_num) and f_num > 0:
|
||||
parsed_followers = int(f_num)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Quarantine suspicious handles with < 10,000 followers to prevent dormant penalties
|
||||
df.loc[df["instagram_followers"] < 10000, "instagram_followers"] = np.nan
|
||||
entry = {
|
||||
"handle": raw_handle if raw_handle and raw_handle.lower() not in ("nan", "none") else None,
|
||||
"followers": parsed_followers
|
||||
}
|
||||
|
||||
# 3. Ingest Kworb Public Streaming Metrics & Daily Velocity
|
||||
print("\n" + "=" * 90)
|
||||
print("PHASE 2: MERGING SIGNALS & ENFORCING RELIABILITY CRITERIA")
|
||||
print("=" * 90)
|
||||
if norm_name:
|
||||
chk_by_name[norm_name] = entry
|
||||
if norm_handle:
|
||||
chk_by_handle[norm_handle] = entry
|
||||
|
||||
kworb = KworbStreamingCollector()
|
||||
kworb.load_aggregator_tables(max_pages=2)
|
||||
print(f"[Loader] Loaded {len(chk)} checkpoints ({sum(1 for v in chk_by_name.values() if v['followers'] is not None)} verified follower records).")
|
||||
|
||||
df["spotify_monthly_listeners"] = df["name"].apply(lambda n: kworb.get_monthly_listeners(n))
|
||||
df["spotify_daily_change"] = df["name"].apply(lambda n: kworb.get_daily_change(n))
|
||||
# 3. Instagram Candidate Verification Engine
|
||||
print("\n[Phase 1] Executing Automated Instagram Verification & Provenance...")
|
||||
verifier = InstagramCandidateVerifier(overrides_path=OVERRIDES_PATH)
|
||||
|
||||
# Audit Provenance Tracking
|
||||
df["youtube_data_source"] = "youtube_data_api_v3"
|
||||
df["spotify_data_source"] = df["spotify_monthly_listeners"].apply(
|
||||
lambda x: "kworb_public_aggregator" if pd.notna(x) else None
|
||||
)
|
||||
df["spotify_status"] = df["spotify_monthly_listeners"].apply(
|
||||
lambda x: "VERIFIED_PUBLIC_AGGREGATOR" if pd.notna(x) else "UNAVAILABLE_IN_TOP_5000"
|
||||
)
|
||||
verified_handles = []
|
||||
verified_followers = []
|
||||
confidence_scores = []
|
||||
verification_statuses = []
|
||||
audit_evidences = []
|
||||
|
||||
for _, row in df.iterrows():
|
||||
a_name = str(row["name"]).strip()
|
||||
norm_a_name = clean_norm_key(a_name)
|
||||
|
||||
# Retrieve candidate handle and followers from checkpoints first
|
||||
matched_chk = chk_by_name.get(norm_a_name)
|
||||
|
||||
candidate_handle = None
|
||||
candidate_followers = None
|
||||
|
||||
if matched_chk:
|
||||
candidate_handle = matched_chk["handle"]
|
||||
candidate_followers = matched_chk["followers"]
|
||||
|
||||
# Fallback via handle normalization
|
||||
if not candidate_handle:
|
||||
row_h = row.get("instagram_handle")
|
||||
if pd.notna(row_h) and str(row_h).strip().lower() not in ("nan", "none", ""):
|
||||
candidate_handle = str(row_h).replace("@", "").strip()
|
||||
norm_row_h = clean_norm_key(candidate_handle)
|
||||
if norm_row_h in chk_by_handle and not candidate_followers:
|
||||
candidate_followers = chk_by_handle[norm_row_h]["followers"]
|
||||
|
||||
# Fallback via existing row followers
|
||||
if candidate_followers is None:
|
||||
r_f = row.get("instagram_followers")
|
||||
if pd.notna(r_f):
|
||||
try:
|
||||
f_num = float(r_f)
|
||||
if not np.isnan(f_num) and f_num > 0:
|
||||
candidate_followers = int(f_num)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
res = verifier.evaluate_candidate(
|
||||
artist_name=a_name,
|
||||
candidate_handle=candidate_handle,
|
||||
follower_count=candidate_followers,
|
||||
youtube_channel_name=row.get("channel_title") if "channel_title" in row else None
|
||||
)
|
||||
|
||||
verified_handles.append(res["handle"])
|
||||
verified_followers.append(res["followers"])
|
||||
confidence_scores.append(res["confidence_score"])
|
||||
verification_statuses.append(res["verification_status"])
|
||||
audit_evidences.append(res["evidence"])
|
||||
|
||||
df["instagram_handle"] = verified_handles
|
||||
df["instagram_followers"] = pd.to_numeric(verified_followers, errors="coerce")
|
||||
df["instagram_confidence_score"] = confidence_scores
|
||||
df["instagram_verification_status"] = verification_statuses
|
||||
df["instagram_audit_evidence"] = audit_evidences
|
||||
df["instagram_data_source"] = df["instagram_followers"].apply(
|
||||
lambda x: "playwright_meta_scrape" if pd.notna(x) else None
|
||||
)
|
||||
|
||||
# Completeness Flag
|
||||
# 4. Spotify Multi-Tier Telemetry Ingestion
|
||||
print("\n[Phase 2] Ingesting Spotify Telemetry (Tier 1: Kworb Aggregator -> Tier 2: Embed Fallback)...")
|
||||
kworb = KworbStreamingCollector()
|
||||
kworb.load_aggregator_tables(max_pages=2)
|
||||
fallback_collector = SpotifyFallbackCollector()
|
||||
|
||||
sp_listeners = []
|
||||
sp_daily = []
|
||||
sp_source = []
|
||||
sp_status = []
|
||||
sp_dates = []
|
||||
|
||||
now_utc = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
for _, row in df.iterrows():
|
||||
name = str(row["name"]).strip()
|
||||
listeners, daily = kworb.get_artist_metrics(name)
|
||||
|
||||
if listeners is not None:
|
||||
sp_listeners.append(listeners)
|
||||
sp_daily.append(daily)
|
||||
sp_source.append("kworb_public_aggregator")
|
||||
sp_status.append("VERIFIED_PUBLIC_AGGREGATOR")
|
||||
sp_dates.append(now_utc)
|
||||
else:
|
||||
sp_id = row.get("spotify_id") or row.get("spotify_uri")
|
||||
if pd.notna(sp_id) and str(sp_id).strip().lower() not in ("nan", "none", ""):
|
||||
fb_res = fallback_collector.fetch_artist_fallback(name, str(sp_id))
|
||||
sp_listeners.append(fb_res["spotify_monthly_listeners"])
|
||||
sp_daily.append(fb_res["spotify_daily_change"])
|
||||
sp_source.append(fb_res["spotify_data_source"])
|
||||
sp_status.append(fb_res["spotify_status"])
|
||||
sp_dates.append(fb_res["collection_date"])
|
||||
else:
|
||||
sp_listeners.append(None)
|
||||
sp_daily.append(None)
|
||||
sp_source.append(None)
|
||||
sp_status.append("UNAVAILABLE_IN_TOP_5000")
|
||||
sp_dates.append(now_utc)
|
||||
|
||||
df["spotify_monthly_listeners"] = pd.to_numeric(sp_listeners, errors="coerce")
|
||||
df["spotify_daily_change"] = pd.to_numeric(sp_daily, errors="coerce")
|
||||
df["spotify_data_source"] = sp_source
|
||||
df["spotify_status"] = sp_status
|
||||
df["spotify_collection_date"] = sp_dates
|
||||
df["youtube_data_source"] = "youtube_data_api_v3"
|
||||
|
||||
df["has_complete_multiplatform_signals"] = (
|
||||
df["production_hiffi_score"].notna() &
|
||||
df["spotify_monthly_listeners"].notna() &
|
||||
df["instagram_followers"].notna()
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# DUAL-TRACK SCORING WITH VELOCITY ENHANCEMENT
|
||||
# -------------------------------------------------------------
|
||||
# Production baseline is strictly preserved
|
||||
# 5. Dual-Track Scoring: Protect YouTube Baseline
|
||||
if "production_hiffi_score" not in df.columns:
|
||||
df["production_hiffi_score"] = 50.0
|
||||
if "production_hiffi_rank" not in df.columns:
|
||||
df["production_hiffi_rank"] = df["production_hiffi_score"].rank(ascending=False, method="min").astype(int)
|
||||
|
||||
# Normalized Component Scores with Bayesian Neutral Anchors
|
||||
# Calculate Experimental HIFFI-XP with Bayesian Median Neutral Anchors
|
||||
yt_norm = min_max_norm(df["production_hiffi_score"], neutral_val=50.0)
|
||||
|
||||
spotify_valid = df["spotify_monthly_listeners"].dropna()
|
||||
spotify_median = (spotify_valid.median() - spotify_valid.min()) / (spotify_valid.max() - spotify_valid.min()) * 100.0 if not spotify_valid.empty else 50.0
|
||||
spotify_norm = min_max_norm(df["spotify_monthly_listeners"], neutral_val=spotify_median)
|
||||
sp_valid = df["spotify_monthly_listeners"].dropna()
|
||||
sp_median = (sp_valid.median() - sp_valid.min()) / (sp_valid.max() - sp_valid.min()) * 100.0 if not sp_valid.empty else 50.0
|
||||
sp_norm = min_max_norm(df["spotify_monthly_listeners"], neutral_val=sp_median)
|
||||
|
||||
ig_valid = df["instagram_followers"].dropna()
|
||||
ig_median = (ig_valid.median() - ig_valid.min()) / (ig_valid.max() - ig_valid.min()) * 100.0 if not ig_valid.empty else 50.0
|
||||
ig_norm = min_max_norm(df["instagram_followers"], neutral_val=ig_median)
|
||||
|
||||
# Daily Velocity Calculation: Ratio of Daily Change to Total Listeners
|
||||
# Missing velocity data defaults to a neutral 0.0 momentum adjustment (no synthetic penalties)
|
||||
# Normalized velocity modifier (bounded [-5.0, +5.0])
|
||||
velocity_ratio = df["spotify_daily_change"] / df["spotify_monthly_listeners"]
|
||||
valid_velocity = velocity_ratio.dropna()
|
||||
|
||||
if not valid_velocity.empty and valid_velocity.std() > 0:
|
||||
norm_velocity = (velocity_ratio - valid_velocity.mean()) / valid_velocity.std()
|
||||
momentum_adjustment = np.clip(norm_velocity.fillna(0.0) * 1.5, -5.0, 5.0)
|
||||
valid_vel = velocity_ratio.dropna()
|
||||
if not valid_vel.empty and valid_vel.std() > 0:
|
||||
norm_vel = (velocity_ratio - valid_vel.mean()) / valid_vel.std()
|
||||
momentum = np.clip(norm_vel.fillna(0.0) * 1.5, -5.0, 5.0)
|
||||
else:
|
||||
momentum_adjustment = 0.0
|
||||
momentum = 0.0
|
||||
|
||||
# Composite HIFFI-XP Score (Bounded strictly between 0.0 and 100.0)
|
||||
raw_xp = (0.45 * yt_norm) + (0.35 * spotify_norm) + (0.20 * ig_norm) + momentum_adjustment
|
||||
raw_xp = (0.45 * yt_norm) + (0.35 * sp_norm) + (0.20 * ig_norm) + momentum
|
||||
df["hiffi_xp_score"] = np.clip(raw_xp, 0.0, 100.0).round(2)
|
||||
df["hiffi_xp_rank"] = df["hiffi_xp_score"].rank(ascending=False, method="min").astype(int)
|
||||
|
||||
# 4. Export Final Audited Dataset
|
||||
# 6. Export Audited Dataset
|
||||
CSV_EXPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_csv(CSV_EXPORT_PATH, index=False)
|
||||
print(f"\nFinal Hardened Multi-Platform Dataset Exported: {CSV_EXPORT_PATH}")
|
||||
|
||||
# 5. Pipeline Data Reliability & Audit Summary
|
||||
verified_spotify = df["spotify_monthly_listeners"].notna().sum()
|
||||
verified_ig = (df["instagram_followers"] >= 10000).sum()
|
||||
complete_signals = df["has_complete_multiplatform_signals"].sum()
|
||||
print(f"\n[Exporter] Final Audited Dataset Exported: {CSV_EXPORT_PATH}")
|
||||
|
||||
# Summary
|
||||
v_spotify = df["spotify_monthly_listeners"].notna().sum()
|
||||
v_ig = (df["instagram_verification_status"].isin(["MANUAL_VERIFIED", "AUTOMATED_VERIFIED"])).sum()
|
||||
print("\n" + "=" * 90)
|
||||
print("PIPELINE DATA RELIABILITY & AUDIT SUMMARY")
|
||||
print("AUDIT SUMMARY (ZERO-IMPUTATION & VERIFIED COVERAGE)")
|
||||
print("=" * 90)
|
||||
print(f"Total Cohort Size: {len(df)}")
|
||||
print(f"Verified Spotify Telemetry: {v_spotify} / {len(df)}")
|
||||
print(f"Verified Instagram Profiles: {v_ig} / {len(df)}")
|
||||
print(f"Complete 3-Platform Coverage: {df['has_complete_multiplatform_signals'].sum()} / {len(df)}")
|
||||
print("=" * 90)
|
||||
print(f"Total Cohort Rows: {len(df)}")
|
||||
print(f"Spotify Verified Telemetry Ingest: {verified_spotify} (Source: kworb_public_aggregator)")
|
||||
print(f"Verified Instagram Reach (>10k): {verified_ig} / {len(df)}")
|
||||
print(f"Complete 3-Platform Signals: {complete_signals} / {len(df)}")
|
||||
print("-" * 90)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user