feat: scale HIFFI pipeline to 500 artists with dual BVI ranking, dark dashboard, and Excel audit export
This commit is contained in:
Srilekha committed
1 parent
5d441ffe7c
commit
9710cf9948
14 files changed
+3017
-358
No files matched your search
@@ -1,91 +1,196 @@
|
||||
"""
|
||||
main.py
|
||||
Production CSV-driven batch pipeline:
|
||||
- Loads seed artists from data/seed_artists_500.csv via RosterLoader
|
||||
- Checkpoint-aware batch execution (saves every CHUNK_SIZE artists)
|
||||
- tqdm progress tracking with ETA
|
||||
- Exports:
|
||||
1. enriched_artists.csv (Full master dataset)
|
||||
2. identity_verification_audit.xlsx (Separate multi-tab Excel workbook)
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
import pandas as pd
|
||||
from dotenv import load_dotenv
|
||||
from tqdm import tqdm
|
||||
|
||||
from collectors.youtube_collector import YouTubeCollector
|
||||
from verifiers.musicbrainz_verifier import MusicBrainzVerifier
|
||||
from verifiers.wikidata_verifier import WikidataVerifier
|
||||
from core.confidence_scorer import ConfidenceScorer
|
||||
from core.ranking_engine import RankingEngine
|
||||
from core.ranking_engine import HIFFIRankingEngine
|
||||
from core.health_detector import ChannelHealthDetector
|
||||
from data.roster_loader import RosterLoader
|
||||
|
||||
def run_pipeline(input_csv: str = "cleaned_artists_inventory.csv", limit: int = 10, output_csv: str = "enriched_artists.csv"):
|
||||
load_dotenv()
|
||||
api_key = os.getenv("YOUTUBE_API_KEY")
|
||||
load_dotenv()
|
||||
|
||||
if not api_key:
|
||||
print("[Notice] No YOUTUBE_API_KEY found in .env. Running YouTube in offline fallback mode.")
|
||||
CSV_ROSTER_PATH = "data/seed_artists_500.csv"
|
||||
OUTPUT_CSV_PATH = "enriched_artists.csv"
|
||||
OUTPUT_EXCEL_PATH = "identity_verification_audit.xlsx"
|
||||
CHUNK_SIZE = 25
|
||||
|
||||
print(f"Loading data from {input_csv}...")
|
||||
df = pd.read_csv(input_csv)
|
||||
|
||||
def export_verification_excel(df: pd.DataFrame, excel_path: str = OUTPUT_EXCEL_PATH):
|
||||
"""
|
||||
Exports a clean, multi-tab Excel workbook dedicated to identity verification audits.
|
||||
"""
|
||||
# Columns relevant for identity verification
|
||||
audit_cols = [
|
||||
"name",
|
||||
"channel_title",
|
||||
"identifier",
|
||||
"confidence_score",
|
||||
"confidence_tier",
|
||||
"status_flag",
|
||||
"mb_verified",
|
||||
"wd_verified",
|
||||
"cross_referenced",
|
||||
]
|
||||
|
||||
# Process only the selected sample batch limit
|
||||
batch_df = df.head(limit).copy()
|
||||
print(f"Processing sample batch of {len(batch_df)} artists...\n")
|
||||
# Filter available columns defensively
|
||||
existing_cols = [c for c in audit_cols if c in df.columns]
|
||||
audit_df = df[existing_cols].copy()
|
||||
|
||||
# Initialize all services
|
||||
yt_collector = YouTubeCollector(api_keys=[api_key] if api_key else None)
|
||||
# Sort by confidence score descending
|
||||
if "confidence_score" in audit_df.columns:
|
||||
audit_df = audit_df.sort_values(by="confidence_score", ascending=False)
|
||||
|
||||
# Flagged subset for low confidence or failed lookups
|
||||
flagged_df = audit_df[
|
||||
(audit_df["confidence_score"] < 0.50)
|
||||
| (audit_df["status_flag"].isin(["FAILED_LOOKUP", "LOOKUP_ERROR"]))
|
||||
].copy()
|
||||
|
||||
with pd.ExcelWriter(excel_path, engine="openpyxl") as writer:
|
||||
audit_df.to_excel(writer, sheet_name="Identity Verification Audit", index=False)
|
||||
flagged_df.to_excel(writer, sheet_name="Flagged & Unresolved", index=False)
|
||||
|
||||
print(f"📊 Separate Identity Verification workbook exported to '{excel_path}'")
|
||||
|
||||
|
||||
def run_batch_pipeline(roster_path: str = CSV_ROSTER_PATH, force_refresh: bool = False):
|
||||
print(f"\n========================================================")
|
||||
print(f" HIFFI PRODUCTION INGESTION PIPELINE")
|
||||
print(f" Source: {roster_path} | Checkpoint Chunk: {CHUNK_SIZE}")
|
||||
print(f"========================================================\n")
|
||||
|
||||
loader = RosterLoader(file_path=roster_path)
|
||||
artists = loader.load_roster()
|
||||
total_artists = len(artists)
|
||||
|
||||
collector = YouTubeCollector()
|
||||
mb_verifier = MusicBrainzVerifier()
|
||||
wd_verifier = WikidataVerifier()
|
||||
scorer = ConfidenceScorer()
|
||||
ranking_engine = RankingEngine()
|
||||
conf_scorer = ConfidenceScorer()
|
||||
ranking_engine = HIFFIRankingEngine()
|
||||
health_detector = ChannelHealthDetector()
|
||||
|
||||
results = []
|
||||
all_records = []
|
||||
|
||||
for idx, row in batch_df.iterrows():
|
||||
name = str(row.get("Name", "")).strip()
|
||||
yt_url = str(row.get("Youtube", "")).strip()
|
||||
print(f"[{idx + 1}/{len(batch_df)}] Analyzing Artist: '{name}'")
|
||||
progress_bar = tqdm(
|
||||
total=total_artists,
|
||||
desc="Processing Roster",
|
||||
unit="artist",
|
||||
ncols=95,
|
||||
bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]"
|
||||
)
|
||||
|
||||
# 1. Collect YouTube 5 metrics
|
||||
yt_data = yt_collector.get_channel_metrics(yt_url, fetch_recent_videos=False)
|
||||
for i, artist in enumerate(artists):
|
||||
name = artist["name"]
|
||||
ident = artist["identifier"]
|
||||
progress_bar.set_postfix_str(f"Analyzing: {name[:18]}")
|
||||
|
||||
# 2. Verify against MusicBrainz
|
||||
mb_data = mb_verifier.verify_artist(name)
|
||||
record = {**artist}
|
||||
|
||||
# 3. Verify against Wikidata
|
||||
wd_data = wd_verifier.verify_artist(name)
|
||||
try:
|
||||
# 1. Fetch channel metrics
|
||||
yt = collector.get_channel_metrics(
|
||||
ident,
|
||||
fetch_recent_videos=True,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
record.update(yt)
|
||||
|
||||
# 4. Compute composite confidence score
|
||||
score_data = scorer.calculate_score(yt_data, mb_data, wd_data)
|
||||
# 2. Verify identity across MusicBrainz & Wikidata
|
||||
mb = mb_verifier.verify_artist(name)
|
||||
wd = wd_verifier.verify_artist(name)
|
||||
|
||||
# Merge results into record
|
||||
results.append({
|
||||
"UID": row.get("UID", ""),
|
||||
"Name": name,
|
||||
"Youtube": yt_url,
|
||||
"Instagram": row.get("Instagram", ""),
|
||||
"Location": row.get("Location", ""),
|
||||
"yt_subscribers": yt_data.get("subscribers", 0),
|
||||
"yt_lifetime_views": yt_data.get("lifetime_views", 0),
|
||||
"yt_video_count": yt_data.get("video_count", 0),
|
||||
"yt_recent_avg_views": yt_data.get("recent_avg_views", 0),
|
||||
"yt_momentum_30d": yt_data.get("momentum_30d", 0.0),
|
||||
"yt_status": yt_data.get("status", ""),
|
||||
"mb_verified": mb_data.get("verified", False),
|
||||
"mb_id": mb_data.get("mbid", ""),
|
||||
"wd_verified": wd_data.get("verified", False),
|
||||
"wd_id": wd_data.get("wikidata_id", ""),
|
||||
"composite_confidence": score_data.get("composite_confidence", 0.0),
|
||||
"confidence_tier": score_data.get("confidence_tier", "Low")
|
||||
})
|
||||
yt_with_name = {**yt, "name": name}
|
||||
conf_result = conf_scorer.calculate_score(
|
||||
yt_metrics=yt_with_name, mb_result=mb, wd_result=wd
|
||||
)
|
||||
|
||||
raw_out_df = pd.DataFrame(results)
|
||||
# Record verification sub-metrics for Excel export
|
||||
record["confidence_score"] = float(conf_result.get("composite_confidence", 0.0))
|
||||
record["confidence_tier"] = conf_result.get("confidence_tier", "Unknown")
|
||||
sub_scores = conf_result.get("sub_scores", {})
|
||||
record["mb_verified"] = bool(sub_scores.get("musicbrainz", 0.0) == 1.0)
|
||||
record["wd_verified"] = bool(sub_scores.get("wikidata", 0.0) == 1.0)
|
||||
record["cross_referenced"] = bool(sub_scores.get("cross_referenced", False))
|
||||
|
||||
# 5. Apply Log-Transformation + Min-Max Scaling Ranking Engine
|
||||
print("\nApplying logarithmic normalization & Min-Max ranking...")
|
||||
out_df = ranking_engine.normalize_metrics(raw_out_df)
|
||||
except Exception as e:
|
||||
record.update({
|
||||
"channel_id": None,
|
||||
"channel_title": None,
|
||||
"subscriber_count": None,
|
||||
"lifetime_views": None,
|
||||
"video_count": None,
|
||||
"recent_avg_views": None,
|
||||
"daily_view_velocity": None,
|
||||
"momentum_30d": None,
|
||||
"status_flag": "LOOKUP_ERROR",
|
||||
"data_complete": 0,
|
||||
"confidence_score": 0.0,
|
||||
"confidence_tier": "Error / Unresolved",
|
||||
"mb_verified": False,
|
||||
"wd_verified": False,
|
||||
"cross_referenced": False,
|
||||
"error_notes": str(e),
|
||||
})
|
||||
|
||||
all_records.append(record)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Intermediate checkpoint every CHUNK_SIZE
|
||||
if (i + 1) % CHUNK_SIZE == 0 or (i + 1) == total_artists:
|
||||
temp_df = pd.DataFrame(all_records)
|
||||
ranked_temp = ranking_engine.score_and_rank(temp_df)
|
||||
health_temp = health_detector.evaluate_dataframe(ranked_temp)
|
||||
health_temp.to_csv(OUTPUT_CSV_PATH, index=False)
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Final scoring and health detection
|
||||
print("\n--- Finalizing Official HIFFI Ranks & Breakout Indices ---")
|
||||
df_raw = pd.DataFrame(all_records)
|
||||
ranked_df = ranking_engine.score_and_rank(df_raw)
|
||||
final_df = health_detector.evaluate_dataframe(ranked_df)
|
||||
|
||||
# 1. Save Master CSV
|
||||
final_df.to_csv(OUTPUT_CSV_PATH, index=False)
|
||||
print(f"✅ Master dataset written to '{OUTPUT_CSV_PATH}'")
|
||||
|
||||
# 2. Save Separate Identity Verification Excel
|
||||
export_verification_excel(final_df, excel_path=OUTPUT_EXCEL_PATH)
|
||||
|
||||
# Summary Leaderboard
|
||||
print("\n--- Top 10 HIFFI Market Leaders ---")
|
||||
hiffi_top10 = final_df[final_df["is_rank_eligible"] == True].sort_values(
|
||||
by="hiffi_rank", ascending=True
|
||||
).head(10)
|
||||
summary_cols = [
|
||||
"hiffi_rank",
|
||||
"name",
|
||||
"hiffi_score",
|
||||
"breakout_rank",
|
||||
"breakout_velocity_index",
|
||||
"daily_view_velocity",
|
||||
"health_status",
|
||||
]
|
||||
print(hiffi_top10[summary_cols].to_string(index=False))
|
||||
|
||||
out_df.to_csv(output_csv, index=False)
|
||||
print(f"Pipeline run complete. Results saved to '{output_csv}'.\n")
|
||||
|
||||
# Display the final ranked overview table
|
||||
summary_cols = ["hiffi_rank", "Name", "hiffi_score", "yt_subscribers", "composite_confidence", "confidence_tier"]
|
||||
print(out_df[summary_cols].to_string(index=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Pipeline")
|
||||
parser.add_argument("--limit", type=int, default=10, help="Number of records to test")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_pipeline(limit=args.limit)
|
||||
run_batch_pipeline(roster_path=CSV_ROSTER_PATH, force_refresh=False)
|
||||
Reference in new issue
Block a user