import os import argparse import pandas as pd from dotenv import load_dotenv from collectors.youtube_collector import YouTubeCollector from verifiers.musicbrainz_verifier import MusicBrainzVerifier from verifiers.wikidata_verifier import WikidataVerifier from core.confidence_scorer import ConfidenceScorer from core.ranking_engine import RankingEngine def run_pipeline(input_csv: str = "cleaned_artists_inventory.csv", limit: int = 10, output_csv: str = "enriched_artists.csv"): load_dotenv() api_key = os.getenv("YOUTUBE_API_KEY") if not api_key: print("[Notice] No YOUTUBE_API_KEY found in .env. Running YouTube in offline fallback mode.") print(f"Loading data from {input_csv}...") df = pd.read_csv(input_csv) # Process only the selected sample batch limit batch_df = df.head(limit).copy() print(f"Processing sample batch of {len(batch_df)} artists...\n") # Initialize all services yt_collector = YouTubeCollector(api_keys=[api_key] if api_key else None) mb_verifier = MusicBrainzVerifier() wd_verifier = WikidataVerifier() scorer = ConfidenceScorer() ranking_engine = RankingEngine() results = [] for idx, row in batch_df.iterrows(): name = str(row.get("Name", "")).strip() yt_url = str(row.get("Youtube", "")).strip() print(f"[{idx + 1}/{len(batch_df)}] Analyzing Artist: '{name}'") # 1. Collect YouTube 5 metrics yt_data = yt_collector.get_channel_metrics(yt_url, fetch_recent_videos=False) # 2. Verify against MusicBrainz mb_data = mb_verifier.verify_artist(name) # 3. Verify against Wikidata wd_data = wd_verifier.verify_artist(name) # 4. Compute composite confidence score score_data = scorer.calculate_score(yt_data, mb_data, wd_data) # Merge results into record results.append({ "UID": row.get("UID", ""), "Name": name, "Youtube": yt_url, "Instagram": row.get("Instagram", ""), "Location": row.get("Location", ""), "yt_subscribers": yt_data.get("subscribers", 0), "yt_lifetime_views": yt_data.get("lifetime_views", 0), "yt_video_count": yt_data.get("video_count", 0), "yt_recent_avg_views": yt_data.get("recent_avg_views", 0), "yt_momentum_30d": yt_data.get("momentum_30d", 0.0), "yt_status": yt_data.get("status", ""), "mb_verified": mb_data.get("verified", False), "mb_id": mb_data.get("mbid", ""), "wd_verified": wd_data.get("verified", False), "wd_id": wd_data.get("wikidata_id", ""), "composite_confidence": score_data.get("composite_confidence", 0.0), "confidence_tier": score_data.get("confidence_tier", "Low") }) raw_out_df = pd.DataFrame(results) # 5. Apply Log-Transformation + Min-Max Scaling Ranking Engine print("\nApplying logarithmic normalization & Min-Max ranking...") out_df = ranking_engine.normalize_metrics(raw_out_df) out_df.to_csv(output_csv, index=False) print(f"Pipeline run complete. Results saved to '{output_csv}'.\n") # Display the final ranked overview table summary_cols = ["hiffi_rank", "Name", "hiffi_score", "yt_subscribers", "composite_confidence", "confidence_tier"] print(out_df[summary_cols].to_string(index=False)) if __name__ == "__main__": parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Pipeline") parser.add_argument("--limit", type=int, default=10, help="Number of records to test") args = parser.parse_args() run_pipeline(limit=args.limit)