checkpoint: working 10-artist batch with log-normalized ranking

main.py

@@ -0,0 +1,91 @@
+import os
+import argparse
+import pandas as pd
+from dotenv import load_dotenv
+
+from collectors.youtube_collector import YouTubeCollector
+from verifiers.musicbrainz_verifier import MusicBrainzVerifier
+from verifiers.wikidata_verifier import WikidataVerifier
+from core.confidence_scorer import ConfidenceScorer
+from core.ranking_engine import RankingEngine
+
+def run_pipeline(input_csv: str = "cleaned_artists_inventory.csv", limit: int = 10, output_csv: str = "enriched_artists.csv"):
+    load_dotenv()
+    api_key = os.getenv("YOUTUBE_API_KEY")
+
+    if not api_key:
+        print("[Notice] No YOUTUBE_API_KEY found in .env. Running YouTube in offline fallback mode.")
+
+    print(f"Loading data from {input_csv}...")
+    df = pd.read_csv(input_csv)
+    
+    # Process only the selected sample batch limit
+    batch_df = df.head(limit).copy()
+    print(f"Processing sample batch of {len(batch_df)} artists...\n")
+
+    # Initialize all services
+    yt_collector = YouTubeCollector(api_keys=[api_key] if api_key else None)
+    mb_verifier = MusicBrainzVerifier()
+    wd_verifier = WikidataVerifier()
+    scorer = ConfidenceScorer()
+    ranking_engine = RankingEngine()
+
+    results = []
+
+    for idx, row in batch_df.iterrows():
+        name = str(row.get("Name", "")).strip()
+        yt_url = str(row.get("Youtube", "")).strip()
+        print(f"[{idx + 1}/{len(batch_df)}] Analyzing Artist: '{name}'")
+
+        # 1. Collect YouTube 5 metrics
+        yt_data = yt_collector.get_channel_metrics(yt_url, fetch_recent_videos=False)
+
+        # 2. Verify against MusicBrainz
+        mb_data = mb_verifier.verify_artist(name)
+
+        # 3. Verify against Wikidata
+        wd_data = wd_verifier.verify_artist(name)
+
+        # 4. Compute composite confidence score
+        score_data = scorer.calculate_score(yt_data, mb_data, wd_data)
+
+        # Merge results into record
+        results.append({
+            "UID": row.get("UID", ""),
+            "Name": name,
+            "Youtube": yt_url,
+            "Instagram": row.get("Instagram", ""),
+            "Location": row.get("Location", ""),
+            "yt_subscribers": yt_data.get("subscribers", 0),
+            "yt_lifetime_views": yt_data.get("lifetime_views", 0),
+            "yt_video_count": yt_data.get("video_count", 0),
+            "yt_recent_avg_views": yt_data.get("recent_avg_views", 0),
+            "yt_momentum_30d": yt_data.get("momentum_30d", 0.0),
+            "yt_status": yt_data.get("status", ""),
+            "mb_verified": mb_data.get("verified", False),
+            "mb_id": mb_data.get("mbid", ""),
+            "wd_verified": wd_data.get("verified", False),
+            "wd_id": wd_data.get("wikidata_id", ""),
+            "composite_confidence": score_data.get("composite_confidence", 0.0),
+            "confidence_tier": score_data.get("confidence_tier", "Low")
+        })
+
+    raw_out_df = pd.DataFrame(results)
+
+    # 5. Apply Log-Transformation + Min-Max Scaling Ranking Engine
+    print("\nApplying logarithmic normalization & Min-Max ranking...")
+    out_df = ranking_engine.normalize_metrics(raw_out_df)
+
+    out_df.to_csv(output_csv, index=False)
+    print(f"Pipeline run complete. Results saved to '{output_csv}'.\n")
+    
+    # Display the final ranked overview table
+    summary_cols = ["hiffi_rank", "Name", "hiffi_score", "yt_subscribers", "composite_confidence", "confidence_tier"]
+    print(out_df[summary_cols].to_string(index=False))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Pipeline")
+    parser.add_argument("--limit", type=int, default=10, help="Number of records to test")
+    args = parser.parse_args()
+
+    run_pipeline(limit=args.limit)