hiffi-artist-intelligence/main.py

"""
main.py
Production CSV-driven batch pipeline:
- Loads seed artists from data/seed_artists_500.csv via RosterLoader
- Checkpoint-aware batch execution (saves every CHUNK_SIZE artists)
- tqdm progress tracking with ETA
- Exports:
    1. enriched_artists.csv (Full master dataset)
    2. identity_verification_audit.xlsx (Separate multi-tab Excel workbook)
"""

import os
import sys
import time
import pandas as pd
from dotenv import load_dotenv
from tqdm import tqdm

from collectors.youtube_collector import YouTubeCollector
from verifiers.musicbrainz_verifier import MusicBrainzVerifier
from verifiers.wikidata_verifier import WikidataVerifier
from core.confidence_scorer import ConfidenceScorer
from core.ranking_engine import HIFFIRankingEngine
from core.health_detector import ChannelHealthDetector
from data.roster_loader import RosterLoader

load_dotenv()

CSV_ROSTER_PATH = "data/seed_artists_500.csv"
OUTPUT_CSV_PATH = "enriched_artists.csv"
OUTPUT_EXCEL_PATH = "identity_verification_audit.xlsx"
CHUNK_SIZE = 25


def export_verification_excel(df: pd.DataFrame, excel_path: str = OUTPUT_EXCEL_PATH):
    """
    Exports a clean, multi-tab Excel workbook dedicated to identity verification audits.
    """
    # Columns relevant for identity verification
    audit_cols = [
        "name",
        "channel_title",
        "identifier",
        "confidence_score",
        "confidence_tier",
        "status_flag",
        "mb_verified",
        "wd_verified",
        "cross_referenced",
    ]
    
    # Filter available columns defensively
    existing_cols = [c for c in audit_cols if c in df.columns]
    audit_df = df[existing_cols].copy()

    # Sort by confidence score descending
    if "confidence_score" in audit_df.columns:
        audit_df = audit_df.sort_values(by="confidence_score", ascending=False)

    # Flagged subset for low confidence or failed lookups
    flagged_df = audit_df[
        (audit_df["confidence_score"] < 0.50)
        | (audit_df["status_flag"].isin(["FAILED_LOOKUP", "LOOKUP_ERROR"]))
    ].copy()

    with pd.ExcelWriter(excel_path, engine="openpyxl") as writer:
        audit_df.to_excel(writer, sheet_name="Identity Verification Audit", index=False)
        flagged_df.to_excel(writer, sheet_name="Flagged & Unresolved", index=False)

    print(f"📊 Separate Identity Verification workbook exported to '{excel_path}'")


def run_batch_pipeline(roster_path: str = CSV_ROSTER_PATH, force_refresh: bool = False):
    print(f"\n========================================================")
    print(f"  HIFFI PRODUCTION INGESTION PIPELINE")
    print(f"  Source: {roster_path} | Checkpoint Chunk: {CHUNK_SIZE}")
    print(f"========================================================\n")

    loader = RosterLoader(file_path=roster_path)
    artists = loader.load_roster()
    total_artists = len(artists)

    collector = YouTubeCollector()
    mb_verifier = MusicBrainzVerifier()
    wd_verifier = WikidataVerifier()
    conf_scorer = ConfidenceScorer()
    ranking_engine = HIFFIRankingEngine()
    health_detector = ChannelHealthDetector()

    all_records = []

    progress_bar = tqdm(
        total=total_artists,
        desc="Processing Roster",
        unit="artist",
        ncols=95,
        bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]"
    )

    for i, artist in enumerate(artists):
        name = artist["name"]
        ident = artist["identifier"]
        progress_bar.set_postfix_str(f"Analyzing: {name[:18]}")

        record = {**artist}

        try:
            # 1. Fetch channel metrics
            yt = collector.get_channel_metrics(
                ident,
                fetch_recent_videos=True,
                force_refresh=force_refresh,
            )
            record.update(yt)

            # 2. Verify identity across MusicBrainz & Wikidata
            mb = mb_verifier.verify_artist(name)
            wd = wd_verifier.verify_artist(name)

            yt_with_name = {**yt, "name": name}
            conf_result = conf_scorer.calculate_score(
                yt_metrics=yt_with_name, mb_result=mb, wd_result=wd
            )

            # Record verification sub-metrics for Excel export
            record["confidence_score"] = float(conf_result.get("composite_confidence", 0.0))
            record["confidence_tier"] = conf_result.get("confidence_tier", "Unknown")
            sub_scores = conf_result.get("sub_scores", {})
            record["mb_verified"] = bool(sub_scores.get("musicbrainz", 0.0) == 1.0)
            record["wd_verified"] = bool(sub_scores.get("wikidata", 0.0) == 1.0)
            record["cross_referenced"] = bool(sub_scores.get("cross_referenced", False))

        except Exception as e:
            record.update({
                "channel_id": None,
                "channel_title": None,
                "subscriber_count": None,
                "lifetime_views": None,
                "video_count": None,
                "recent_avg_views": None,
                "daily_view_velocity": None,
                "momentum_30d": None,
                "status_flag": "LOOKUP_ERROR",
                "data_complete": 0,
                "confidence_score": 0.0,
                "confidence_tier": "Error / Unresolved",
                "mb_verified": False,
                "wd_verified": False,
                "cross_referenced": False,
                "error_notes": str(e),
            })

        all_records.append(record)
        progress_bar.update(1)

        # Intermediate checkpoint every CHUNK_SIZE
        if (i + 1) % CHUNK_SIZE == 0 or (i + 1) == total_artists:
            temp_df = pd.DataFrame(all_records)
            ranked_temp = ranking_engine.score_and_rank(temp_df)
            health_temp = health_detector.evaluate_dataframe(ranked_temp)
            health_temp.to_csv(OUTPUT_CSV_PATH, index=False)

    progress_bar.close()

    # Final scoring and health detection
    print("\n--- Finalizing Official HIFFI Ranks & Breakout Indices ---")
    df_raw = pd.DataFrame(all_records)
    ranked_df = ranking_engine.score_and_rank(df_raw)
    final_df = health_detector.evaluate_dataframe(ranked_df)

    # 1. Save Master CSV
    final_df.to_csv(OUTPUT_CSV_PATH, index=False)
    print(f"✅ Master dataset written to '{OUTPUT_CSV_PATH}'")

    # 2. Save Separate Identity Verification Excel
    export_verification_excel(final_df, excel_path=OUTPUT_EXCEL_PATH)

    # Summary Leaderboard
    print("\n--- Top 10 HIFFI Market Leaders ---")
    hiffi_top10 = final_df[final_df["is_rank_eligible"] == True].sort_values(
        by="hiffi_rank", ascending=True
    ).head(10)
    summary_cols = [
        "hiffi_rank",
        "name",
        "hiffi_score",
        "breakout_rank",
        "breakout_velocity_index",
        "daily_view_velocity",
        "health_status",
    ]
    print(hiffi_top10[summary_cols].to_string(index=False))


if __name__ == "__main__":
    run_batch_pipeline(roster_path=CSV_ROSTER_PATH, force_refresh=False)