91 lines
3.6 KiB
Python
91 lines
3.6 KiB
Python
|
import os
|
||
|
|
import argparse
|
||
|
|
import pandas as pd
|
||
|
|
from dotenv import load_dotenv
|
||
|
|
|
||
|
|
from collectors.youtube_collector import YouTubeCollector
|
||
|
|
from verifiers.musicbrainz_verifier import MusicBrainzVerifier
|
||
|
|
from verifiers.wikidata_verifier import WikidataVerifier
|
||
|
|
from core.confidence_scorer import ConfidenceScorer
|
||
|
|
from core.ranking_engine import RankingEngine
|
||
|
|
|
||
|
|
def run_pipeline(input_csv: str = "cleaned_artists_inventory.csv", limit: int = 10, output_csv: str = "enriched_artists.csv"):
|
||
|
|
load_dotenv()
|
||
|
|
api_key = os.getenv("YOUTUBE_API_KEY")
|
||
|
|
|
||
|
|
if not api_key:
|
||
|
|
print("[Notice] No YOUTUBE_API_KEY found in .env. Running YouTube in offline fallback mode.")
|
||
|
|
|
||
|
|
print(f"Loading data from {input_csv}...")
|
||
|
|
df = pd.read_csv(input_csv)
|
||
|
|
|
||
|
|
# Process only the selected sample batch limit
|
||
|
|
batch_df = df.head(limit).copy()
|
||
|
|
print(f"Processing sample batch of {len(batch_df)} artists...\n")
|
||
|
|
|
||
|
|
# Initialize all services
|
||
|
|
yt_collector = YouTubeCollector(api_keys=[api_key] if api_key else None)
|
||
|
|
mb_verifier = MusicBrainzVerifier()
|
||
|
|
wd_verifier = WikidataVerifier()
|
||
|
|
scorer = ConfidenceScorer()
|
||
|
|
ranking_engine = RankingEngine()
|
||
|
|
|
||
|
|
results = []
|
||
|
|
|
||
|
|
for idx, row in batch_df.iterrows():
|
||
|
|
name = str(row.get("Name", "")).strip()
|
||
|
|
yt_url = str(row.get("Youtube", "")).strip()
|
||
|
|
print(f"[{idx + 1}/{len(batch_df)}] Analyzing Artist: '{name}'")
|
||
|
|
|
||
|
|
# 1. Collect YouTube 5 metrics
|
||
|
|
yt_data = yt_collector.get_channel_metrics(yt_url, fetch_recent_videos=False)
|
||
|
|
|
||
|
|
# 2. Verify against MusicBrainz
|
||
|
|
mb_data = mb_verifier.verify_artist(name)
|
||
|
|
|
||
|
|
# 3. Verify against Wikidata
|
||
|
|
wd_data = wd_verifier.verify_artist(name)
|
||
|
|
|
||
|
|
# 4. Compute composite confidence score
|
||
|
|
score_data = scorer.calculate_score(yt_data, mb_data, wd_data)
|
||
|
|
|
||
|
|
# Merge results into record
|
||
|
|
results.append({
|
||
|
|
"UID": row.get("UID", ""),
|
||
|
|
"Name": name,
|
||
|
|
"Youtube": yt_url,
|
||
|
|
"Instagram": row.get("Instagram", ""),
|
||
|
|
"Location": row.get("Location", ""),
|
||
|
|
"yt_subscribers": yt_data.get("subscribers", 0),
|
||
|
|
"yt_lifetime_views": yt_data.get("lifetime_views", 0),
|
||
|
|
"yt_video_count": yt_data.get("video_count", 0),
|
||
|
|
"yt_recent_avg_views": yt_data.get("recent_avg_views", 0),
|
||
|
|
"yt_momentum_30d": yt_data.get("momentum_30d", 0.0),
|
||
|
|
"yt_status": yt_data.get("status", ""),
|
||
|
|
"mb_verified": mb_data.get("verified", False),
|
||
|
|
"mb_id": mb_data.get("mbid", ""),
|
||
|
|
"wd_verified": wd_data.get("verified", False),
|
||
|
|
"wd_id": wd_data.get("wikidata_id", ""),
|
||
|
|
"composite_confidence": score_data.get("composite_confidence", 0.0),
|
||
|
|
"confidence_tier": score_data.get("confidence_tier", "Low")
|
||
|
|
})
|
||
|
|
|
||
|
|
raw_out_df = pd.DataFrame(results)
|
||
|
|
|
||
|
|
# 5. Apply Log-Transformation + Min-Max Scaling Ranking Engine
|
||
|
|
print("\nApplying logarithmic normalization & Min-Max ranking...")
|
||
|
|
out_df = ranking_engine.normalize_metrics(raw_out_df)
|
||
|
|
|
||
|
|
out_df.to_csv(output_csv, index=False)
|
||
|
|
print(f"Pipeline run complete. Results saved to '{output_csv}'.\n")
|
||
|
|
|
||
|
|
# Display the final ranked overview table
|
||
|
|
summary_cols = ["hiffi_rank", "Name", "hiffi_score", "yt_subscribers", "composite_confidence", "confidence_tier"]
|
||
|
|
print(out_df[summary_cols].to_string(index=False))
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Pipeline")
|
||
|
|
parser.add_argument("--limit", type=int, default=10, help="Number of records to test")
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
run_pipeline(limit=args.limit)
|