""" main.py Production CSV-driven batch pipeline: - Loads seed artists from data/seed_artists_500.csv via RosterLoader - Checkpoint-aware batch execution (saves every CHUNK_SIZE artists) - tqdm progress tracking with ETA - Exports: 1. enriched_artists.csv (Full master dataset) 2. identity_verification_audit.xlsx (Separate multi-tab Excel workbook) """ import os import sys import time import pandas as pd from dotenv import load_dotenv from tqdm import tqdm from collectors.youtube_collector import YouTubeCollector from verifiers.musicbrainz_verifier import MusicBrainzVerifier from verifiers.wikidata_verifier import WikidataVerifier from core.confidence_scorer import ConfidenceScorer from core.ranking_engine import HIFFIRankingEngine from core.health_detector import ChannelHealthDetector from data.roster_loader import RosterLoader load_dotenv() CSV_ROSTER_PATH = "data/seed_artists_500.csv" OUTPUT_CSV_PATH = "enriched_artists.csv" OUTPUT_EXCEL_PATH = "identity_verification_audit.xlsx" CHUNK_SIZE = 25 def export_verification_excel(df: pd.DataFrame, excel_path: str = OUTPUT_EXCEL_PATH): """ Exports a clean, multi-tab Excel workbook dedicated to identity verification audits. """ # Columns relevant for identity verification audit_cols = [ "name", "channel_title", "identifier", "confidence_score", "confidence_tier", "status_flag", "mb_verified", "wd_verified", "cross_referenced", ] # Filter available columns defensively existing_cols = [c for c in audit_cols if c in df.columns] audit_df = df[existing_cols].copy() # Sort by confidence score descending if "confidence_score" in audit_df.columns: audit_df = audit_df.sort_values(by="confidence_score", ascending=False) # Flagged subset for low confidence or failed lookups flagged_df = audit_df[ (audit_df["confidence_score"] < 0.50) | (audit_df["status_flag"].isin(["FAILED_LOOKUP", "LOOKUP_ERROR"])) ].copy() with pd.ExcelWriter(excel_path, engine="openpyxl") as writer: audit_df.to_excel(writer, sheet_name="Identity Verification Audit", index=False) flagged_df.to_excel(writer, sheet_name="Flagged & Unresolved", index=False) print(f"📊 Separate Identity Verification workbook exported to '{excel_path}'") def run_batch_pipeline(roster_path: str = CSV_ROSTER_PATH, force_refresh: bool = False): print(f"\n========================================================") print(f" HIFFI PRODUCTION INGESTION PIPELINE") print(f" Source: {roster_path} | Checkpoint Chunk: {CHUNK_SIZE}") print(f"========================================================\n") loader = RosterLoader(file_path=roster_path) artists = loader.load_roster() total_artists = len(artists) collector = YouTubeCollector() mb_verifier = MusicBrainzVerifier() wd_verifier = WikidataVerifier() conf_scorer = ConfidenceScorer() ranking_engine = HIFFIRankingEngine() health_detector = ChannelHealthDetector() all_records = [] progress_bar = tqdm( total=total_artists, desc="Processing Roster", unit="artist", ncols=95, bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]" ) for i, artist in enumerate(artists): name = artist["name"] ident = artist["identifier"] progress_bar.set_postfix_str(f"Analyzing: {name[:18]}") record = {**artist} try: # 1. Fetch channel metrics yt = collector.get_channel_metrics( ident, fetch_recent_videos=True, force_refresh=force_refresh, ) record.update(yt) # 2. Verify identity across MusicBrainz & Wikidata mb = mb_verifier.verify_artist(name) wd = wd_verifier.verify_artist(name) yt_with_name = {**yt, "name": name} conf_result = conf_scorer.calculate_score( yt_metrics=yt_with_name, mb_result=mb, wd_result=wd ) # Record verification sub-metrics for Excel export record["confidence_score"] = float(conf_result.get("composite_confidence", 0.0)) record["confidence_tier"] = conf_result.get("confidence_tier", "Unknown") sub_scores = conf_result.get("sub_scores", {}) record["mb_verified"] = bool(sub_scores.get("musicbrainz", 0.0) == 1.0) record["wd_verified"] = bool(sub_scores.get("wikidata", 0.0) == 1.0) record["cross_referenced"] = bool(sub_scores.get("cross_referenced", False)) except Exception as e: record.update({ "channel_id": None, "channel_title": None, "subscriber_count": None, "lifetime_views": None, "video_count": None, "recent_avg_views": None, "daily_view_velocity": None, "momentum_30d": None, "status_flag": "LOOKUP_ERROR", "data_complete": 0, "confidence_score": 0.0, "confidence_tier": "Error / Unresolved", "mb_verified": False, "wd_verified": False, "cross_referenced": False, "error_notes": str(e), }) all_records.append(record) progress_bar.update(1) # Intermediate checkpoint every CHUNK_SIZE if (i + 1) % CHUNK_SIZE == 0 or (i + 1) == total_artists: temp_df = pd.DataFrame(all_records) ranked_temp = ranking_engine.score_and_rank(temp_df) health_temp = health_detector.evaluate_dataframe(ranked_temp) health_temp.to_csv(OUTPUT_CSV_PATH, index=False) progress_bar.close() # Final scoring and health detection print("\n--- Finalizing Official HIFFI Ranks & Breakout Indices ---") df_raw = pd.DataFrame(all_records) ranked_df = ranking_engine.score_and_rank(df_raw) final_df = health_detector.evaluate_dataframe(ranked_df) # 1. Save Master CSV final_df.to_csv(OUTPUT_CSV_PATH, index=False) print(f"✅ Master dataset written to '{OUTPUT_CSV_PATH}'") # 2. Save Separate Identity Verification Excel export_verification_excel(final_df, excel_path=OUTPUT_EXCEL_PATH) # Summary Leaderboard print("\n--- Top 10 HIFFI Market Leaders ---") hiffi_top10 = final_df[final_df["is_rank_eligible"] == True].sort_values( by="hiffi_rank", ascending=True ).head(10) summary_cols = [ "hiffi_rank", "name", "hiffi_score", "breakout_rank", "breakout_velocity_index", "daily_view_velocity", "health_status", ] print(hiffi_top10[summary_cols].to_string(index=False)) if __name__ == "__main__": run_batch_pipeline(roster_path=CSV_ROSTER_PATH, force_refresh=False)