checkpoint: working 10-artist batch with log-normalized ranking

collectors/youtube_collector.py

@@ -0,0 +1,192 @@
+import os
+import re
+import json
+import sqlite3
+from pathlib import Path
+from datetime import datetime, timezone, timedelta
+from urllib.parse import urlparse
+from typing import Dict, Any, Optional
+
+try:
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+
+class YouTubeCollector:
+    def __init__(self, api_keys: list[str] = None, cache_db: str = ".cache/youtube_metrics.sqlite"):
+        self.api_keys = [k.strip() for k in (api_keys or []) if k.strip()]
+        self.cache_db = Path(cache_db)
+        self.cache_db.parent.mkdir(parents=True, exist_ok=True)
+        self._init_db()
+        self.client = None
+        if self.api_keys and build:
+            try:
+                self.client = build("youtube", "v3", developerKey=self.api_keys[0], cache_discovery=False)
+            except Exception as e:
+                print(f"[Warning] Failed to initialize YouTube API client: {e}")
+
+    def _init_db(self):
+        with sqlite3.connect(self.cache_db) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS channel_metrics (
+                    channel_id TEXT PRIMARY KEY,
+                    title TEXT,
+                    subscribers INTEGER,
+                    lifetime_views INTEGER,
+                    video_count INTEGER,
+                    recent_avg_views INTEGER,
+                    momentum_30d REAL,
+                    raw_json TEXT,
+                    updated_at TEXT
+                )
+            """)
+
+    def _get_cached(self, channel_id: str) -> Optional[Dict[str, Any]]:
+        with sqlite3.connect(self.cache_db) as conn:
+            conn.row_factory = sqlite3.Row
+            row = conn.execute("SELECT * FROM channel_metrics WHERE channel_id = ?", (channel_id,)).fetchone()
+            if row:
+                return dict(row)
+        return None
+
+    def _save_cache(self, metrics: Dict[str, Any]):
+        with sqlite3.connect(self.cache_db) as conn:
+            conn.execute("""
+                INSERT OR REPLACE INTO channel_metrics (
+                    channel_id, title, subscribers, lifetime_views, video_count,
+                    recent_avg_views, momentum_30d, raw_json, updated_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (
+                metrics["channel_id"],
+                metrics["title"],
+                metrics["subscribers"],
+                metrics["lifetime_views"],
+                metrics["video_count"],
+                metrics["recent_avg_views"],
+                metrics["momentum_30d"],
+                json.dumps(metrics),
+                datetime.now(timezone.utc).isoformat()
+            ))
+
+    def extract_channel_id(self, url: str) -> str:
+        """Extract UC... channel ID from standard YouTube URL."""
+        if not url:
+            return ""
+        match = re.search(r"channel/(UC[a-zA-Z0-9_-]{22})", url)
+        if match:
+            return match.group(1)
+        return url.strip().rstrip("/").split("/")[-1]
+
+    def get_channel_metrics(self, youtube_url: str, fetch_recent_videos: bool = False) -> Dict[str, Any]:
+        channel_id = self.extract_channel_id(youtube_url)
+        empty = {
+            "channel_id": channel_id,
+            "title": "",
+            "subscribers": 0,
+            "lifetime_views": 0,
+            "video_count": 0,
+            "recent_avg_views": 0,
+            "momentum_30d": 0.0,
+            "status": "unresolved"
+        }
+        if not channel_id:
+            empty["status"] = "empty_url"
+            return empty
+
+        # 1. Check local cache first (saves API quota)
+        cached = self._get_cached(channel_id)
+        if cached:
+            cached["status"] = "cached"
+            return cached
+
+        # 2. If client is not available (zero API keys), return baseline
+        if not self.client:
+            empty["status"] = "no_api_key"
+            return empty
+
+        try:
+            # 3. Call channels.list for statistics and contentDetails
+            resp = self.client.channels().list(
+                part="snippet,statistics,contentDetails",
+                id=channel_id
+            ).execute()
+
+            items = resp.get("items") or []
+            if not items:
+                empty["status"] = "not_found"
+                return empty
+
+            item = items[0]
+            snippet = item.get("snippet", {})
+            stats = item.get("statistics", {})
+            content = item.get("contentDetails", {})
+
+            metrics = {
+                "channel_id": channel_id,
+                "title": snippet.get("title", ""),
+                "subscribers": int(stats.get("subscriberCount") or 0),
+                "lifetime_views": int(stats.get("viewCount") or 0),
+                "video_count": int(stats.get("videoCount") or 0),
+                "recent_avg_views": 0,
+                "momentum_30d": 0.0,
+                "status": "live_api"
+            }
+
+            # 4. Optional: fetch recent video metrics for average views & momentum
+            uploads_id = content.get("relatedPlaylists", {}).get("uploads")
+            if fetch_recent_videos and uploads_id:
+                recent_views, momentum = self._calc_recent_metrics(uploads_id, metrics["lifetime_views"])
+                metrics["recent_avg_views"] = recent_views
+                metrics["momentum_30d"] = momentum
+
+            self._save_cache(metrics)
+            return metrics
+
+        except Exception as e:
+            empty["status"] = f"error: {str(e)[:100]}"
+            return empty
+
+    def _calc_recent_metrics(self, uploads_playlist_id: str, lifetime_views: int) -> tuple[int, float]:
+        """Fetch up to 10 latest uploads to compute average views and 30d momentum."""
+        try:
+            playlist_resp = self.client.playlistItems().list(
+                part="contentDetails",
+                playlistId=uploads_playlist_id,
+                maxResults=10
+            ).execute()
+
+            video_ids = [i["contentDetails"]["videoId"] for i in playlist_resp.get("items", [])]
+            if not video_ids:
+                return 0, 0.0
+
+            video_resp = self.client.videos().list(
+                part="statistics,snippet",
+                id=",".join(video_ids)
+            ).execute()
+
+            now = datetime.now(timezone.utc)
+            thirty_days_ago = now - timedelta(days=30)
+            
+            views = []
+            views_30d = 0
+
+            for v in video_resp.get("items", []):
+                v_views = int(v.get("statistics", {}).get("viewCount") or 0)
+                views.append(v_views)
+                pub_date_str = v.get("snippet", {}).get("publishedAt", "")
+                if pub_date_str:
+                    pub_date = datetime.fromisoformat(pub_date_str.replace("Z", "+00:00"))
+                    if pub_date >= thirty_days_ago:
+                        views_30d += v_views
+
+            avg_views = int(sum(views) / len(views)) if views else 0
+            momentum = round((views_30d / lifetime_views) * 100, 3) if lifetime_views > 0 else 0.0
+            return avg_views, momentum
+        except Exception:
+            return 0, 0.0
+
+if __name__ == "__main__":
+    collector = YouTubeCollector()
+    test_res = collector.get_channel_metrics("https://www.youtube.com/channel/UCBOxSirnH1-_-xKtqX051lg")
+    print("Test run result:")
+    print(test_res)

core/confidence_scorer.py

@@ -0,0 +1,77 @@
+from typing import Dict, Any
+
+class ConfidenceScorer:
+    """
+    Computes a weighted authenticity and quality score across multiple data sources:
+    - YouTube footprint: 40%
+    - MusicBrainz catalog: 35%
+    - Wikidata entity: 25%
+    """
+    def __init__(self, yt_weight: float = 0.40, mb_weight: float = 0.35, wd_weight: float = 0.25):
+        self.w_yt = yt_weight
+        self.w_mb = mb_weight
+        self.w_wd = wd_weight
+
+    def calculate_score(
+        self,
+        yt_metrics: Dict[str, Any],
+        mb_result: Dict[str, Any],
+        wd_result: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        # 1. YouTube sub-score (0.0 to 1.0)
+        yt_score = 0.0
+        if yt_metrics.get("status") in ["live_api", "cached"]:
+            subs = yt_metrics.get("subscribers", 0)
+            views = yt_metrics.get("lifetime_views", 0)
+            vids = yt_metrics.get("video_count", 0)
+
+            # Baseline 0.5 for a verified channel existence
+            yt_score = 0.5
+            if subs >= 1000:
+                yt_score += 0.2
+            elif subs > 0:
+                yt_score += 0.1
+
+            if views >= 50000:
+                yt_score += 0.2
+            elif views > 0:
+                yt_score += 0.1
+
+            if vids >= 5:
+                yt_score += 0.1
+
+        # 2. MusicBrainz sub-score
+        mb_score = mb_result.get("confidence", 0.0) if mb_result.get("verified") else 0.0
+
+        # 3. Wikidata sub-score
+        wd_score = wd_result.get("confidence", 0.0) if wd_result.get("verified") else 0.0
+
+        # Aggregated composite score
+        composite = (yt_score * self.w_yt) + (mb_score * self.w_mb) + (wd_score * self.w_wd)
+        composite = round(min(1.0, max(0.0, composite)), 2)
+
+        # Confidence category
+        if composite >= 0.75:
+            tier = "High"
+        elif composite >= 0.45:
+            tier = "Medium"
+        else:
+            tier = "Low / Emerging"
+
+        return {
+            "composite_confidence": composite,
+            "confidence_tier": tier,
+            "sub_scores": {
+                "youtube": round(yt_score, 2),
+                "musicbrainz": round(mb_score, 2),
+                "wikidata": round(wd_score, 2)
+            }
+        }
+
+if __name__ == "__main__":
+    scorer = ConfidenceScorer()
+    dummy_yt = {"status": "live_api", "subscribers": 5000, "lifetime_views": 250000, "video_count": 18}
+    dummy_mb = {"verified": True, "confidence": 1.0}
+    dummy_wd = {"verified": True, "confidence": 1.0}
+    print("Test Scorer Output:")
+    print(scorer.calculate_score(dummy_yt, dummy_mb, dummy_wd))

core/deduplication.py

@@ -0,0 +1,59 @@
+import re
+import pandas as pd
+from urllib.parse import urlparse
+
+def clean_url(url: str) -> str:
+    """Normalize URLs by stripping query params, trailing slashes, and spaces."""
+    if not isinstance(url, str) or not url.strip() or url.strip().lower() in ["nan", "none", ""]:
+        return ""
+    url = url.strip().rstrip("/")
+    if not re.match(r"^https?://", url, re.I):
+        url = "https://" + url
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}{parsed.path}".rstrip("/")
+
+def extract_channel_key(yt_url: str) -> str:
+    """Extract unique channel ID or handle to catch cross-name channel duplicates."""
+    clean = clean_url(yt_url).lower()
+    if not clean:
+        return ""
+    parts = [p for p in urlparse(clean).path.split("/") if p]
+    if not parts:
+        return ""
+    return parts[-1]
+
+def deduplicate_inventory(input_path: str = "artist_inventory_artists.csv") -> pd.DataFrame:
+    print(f"Loading {input_path}...")
+    df = pd.read_csv(input_path)
+    initial_count = len(df)
+    print(f"Initial raw rows: {initial_count}")
+
+    # 1. Clean strings and URLs
+    for col in ["Name", "Location"]:
+        df[col] = df[col].fillna("").astype(str).str.strip()
+
+    for col in ["Instagram", "Youtube", "TikTok", "Facebook"]:
+        df[col] = df[col].apply(clean_url)
+
+    # 2. Generate normalized match keys
+    df["clean_name"] = df["Name"].str.lower().str.replace(r"[^\w\s]", "", regex=True).str.strip()
+    df["yt_key"] = df["Youtube"].apply(extract_channel_key)
+
+    # 3. Drop exact duplicates on Name + YouTube
+    df_dedup = df.drop_duplicates(subset=["clean_name", "yt_key"], keep="first").copy()
+    
+    # 4. Check for duplicate YouTube channels across different names
+    yt_conflicts = df_dedup[df_dedup["yt_key"] != ""].duplicated(subset=["yt_key"], keep=False).sum()
+    print(f"Rows with shared YouTube channels across different names: {yt_conflicts}")
+
+    dropped = initial_count - len(df_dedup)
+    print(f"Deduplicated count: {len(df_dedup)} (Dropped {dropped} duplicate rows)")
+
+    # Clean temporary helper columns before returning
+    df_dedup.drop(columns=["clean_name", "yt_key"], inplace=True)
+    return df_dedup
+
+if __name__ == "__main__":
+    cleaned_df = deduplicate_inventory()
+    cleaned_df.to_csv("cleaned_artists_inventory.csv", index=False)
+    print("Saved cleaned data to cleaned_artists_inventory.csv")

core/ranking_engine.py

@@ -0,0 +1,83 @@
+import numpy as np
+import pandas as pd
+from typing import Dict, Any
+
+class RankingEngine:
+    """
+    Computes equitable artist rankings using Log Transformation (log1p)
+    and Min-Max Scaling across skewed social/streaming distribution metrics.
+    """
+    DEFAULT_WEIGHTS = {
+        "norm_subscribers": 0.30,
+        "norm_lifetime_views": 0.25,
+        "norm_recent_avg_views": 0.25,
+        "norm_video_count": 0.10,
+        "norm_momentum_30d": 0.10
+    }
+
+    def __init__(self, weights: Dict[str, float] = None):
+        self.weights = weights or self.DEFAULT_WEIGHTS
+
+    def normalize_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Applies log10(1 + x) and Min-Max scaling to compress power-law tails."""
+        df_scored = df.copy()
+
+        metrics_to_transform = [
+            ("yt_subscribers", "norm_subscribers"),
+            ("yt_lifetime_views", "norm_lifetime_views"),
+            ("yt_video_count", "norm_video_count"),
+            ("yt_recent_avg_views", "norm_recent_avg_views"),
+            ("yt_momentum_30d", "norm_momentum_30d")
+        ]
+
+        for raw_col, norm_col in metrics_to_transform:
+            if raw_col not in df_scored.columns:
+                df_scored[norm_col] = 0.0
+                continue
+
+            # Step 1: Handle NaNs and negative values, then apply Log10(1 + x)
+            series = pd.to_numeric(df_scored[raw_col], errors="coerce").fillna(0).clip(lower=0)
+            log_series = np.log10(1 + series)
+
+            min_val = log_series.min()
+            max_val = log_series.max()
+
+            # Step 2: Min-Max Normalization to scale [0.0, 1.0]
+            if max_val > min_val:
+                df_scored[norm_col] = (log_series - min_val) / (max_val - min_val)
+            else:
+                df_scored[norm_col] = 0.0
+
+        # Step 3: Compute weighted base score
+        df_scored["base_rank_score"] = sum(
+            df_scored[col] * weight for col, weight in self.weights.items()
+        )
+
+        # Step 4: Multiply by verification confidence to penalize unverified or fake profiles
+        # Artists with verified MusicBrainz/Wikidata and legitimate channels get full weight
+        confidence = df_scored.get("composite_confidence", pd.Series(1.0, index=df_scored.index))
+        # Ensure a minimum floor of 0.2 so emerging artists are ranked on data, not dropped completely
+        confidence_multiplier = 0.2 + (0.8 * confidence)
+        
+        # Step 5: Final HIFFI Score on a clean 0 to 100 scale
+        df_scored["hiffi_score"] = (df_scored["base_rank_score"] * confidence_multiplier * 100).round(2)
+        
+        # Sort and assign explicit rank ordinal
+        df_scored = df_scored.sort_values(by="hiffi_score", ascending=False).reset_index(drop=True)
+        df_scored["hiffi_rank"] = df_scored.index + 1
+
+        return df_scored
+
+if __name__ == "__main__":
+    # Test script with dummy mock distribution
+    sample_data = pd.DataFrame([
+        {"Name": "Global Superstar", "yt_subscribers": 15000000, "yt_lifetime_views": 8500000000, "yt_video_count": 120, "yt_recent_avg_views": 2500000, "yt_momentum_30d": 1.2, "composite_confidence": 0.95},
+        {"Name": "Mid-tier Rising", "yt_subscribers": 250000, "yt_lifetime_views": 45000000, "yt_video_count": 45, "yt_recent_avg_views": 180000, "yt_momentum_30d": 5.4, "composite_confidence": 0.85},
+        {"Name": "Emerging Talent", "yt_subscribers": 12000, "yt_lifetime_views": 850000, "yt_video_count": 14, "yt_recent_avg_views": 45000, "yt_momentum_30d": 12.8, "composite_confidence": 0.60},
+        {"Name": "Dormant / Inactive", "yt_subscribers": 500, "yt_lifetime_views": 15000, "yt_video_count": 3, "yt_recent_avg_views": 50, "yt_momentum_30d": 0.0, "composite_confidence": 0.35}
+    ])
+    
+    engine = RankingEngine()
+    ranked = engine.normalize_metrics(sample_data)
+    print("Ranked Test Artists with Log & Min-Max Transformation:")
+    print(ranked[["hiffi_rank", "Name", "hiffi_score", "norm_subscribers", "norm_lifetime_views"]])

enriched_artists.csv

@@ -0,0 +1,11 @@
+UID,Name,Youtube,Instagram,Location,yt_subscribers,yt_lifetime_views,yt_video_count,yt_recent_avg_views,yt_momentum_30d,yt_status,mb_verified,mb_id,wd_verified,wd_id,composite_confidence,confidence_tier,norm_subscribers,norm_lifetime_views,norm_video_count,norm_recent_avg_views,norm_momentum_30d,base_rank_score,hiffi_score,hiffi_rank
+9f76913bf5a8996b14c37563fdff9751,$ILKMONEY,https://www.youtube.com/channel/UCA1PZKy6XNQ-TsXWbQW2vyw,https://instagram.com/silkbooty,,46900,14878731,15,0,0.0,cached,True,309da629-06af-4453-85f4-9c1868a2fbbc,True,Q136428352,0.97,High,0.8776752758308687,0.9062496400532577,0.6717951158281678,0.0,0.0,0.5570445043453918,54.37,1
+4f9cc677d46d75f9709608f0668442fe,$hyfromdatre,https://www.youtube.com/channel/UCHRl8OQoAAUghnEJmsoiP5A,https://instagram.com/shyfromdatre,,210000,82140745,33,0,0.0,cached,True,db59266e-cd49-4f8c-98d2-e4b630251135,False,,0.75,High,1.0,1.0,0.8544331721773178,0.0,0.0,0.6354433172177318,50.84,2
+fb669f1ff32d3a7bbdef274872cbe0fd,+/-,https://www.youtube.com/channel/UCBOxSirnH1-_-xKtqX051lg,https://instagram.com/plusminusnyc,New York,739,164806,31,3477,0.0,cached,False,,False,Q260387,0.36,Low / Emerging,0.539104168005253,0.6591619178536015,0.8397438947852097,1.0,0.0,0.6604961193434973,32.23,3
+0f8e17a71adc4b60b61f9bf8f09605fb,$hoey,https://www.youtube.com/channel/UCkRNQjK0d70NqvLI57JmfeA,https://instagram.com/shoeyrussell,,594,653569,10,0,0.0,cached,False,1144f22d-fb1f-41a7-9822-05dab5edff25,False,,0.36,Low / Emerging,0.5213080735653491,0.7347588629481051,0.5810073162355173,0.0,0.0,0.3981828694301827,19.43,4
+405c893b17cdd7a486fc074f722671ab,$eLL,https://www.youtube.com/channel/UCVhJhIHih93-X-NzZeXX6bA,https://instagram.com/localcel_,,421,56299,61,0,0.0,cached,False,735f2b10-a267-4cd2-990f-f91702be9130,False,Q114274902,0.36,Low / Emerging,0.493273817829613,0.6002240439751241,1.0,0.0,0.0,0.39803815634266493,19.42,5
+e6351059c8846a51fe492ebb6365e4b0,$hamrock,https://www.youtube.com/channel/UCenklnIchLhgF2ELHfLqpmQ,,,125,33862,0,0,0.0,cached,True,771e21e8-78f6-4d06-a495-ddcaa08b469b,False,,0.63,Medium,0.3946417110342386,0.5723282415425062,0.0,0.0,0.0,0.2614745736958981,18.41,6
+c977cafbeabf67c0956bb0d1ce0188d7,$avage,https://www.youtube.com/channel/UCxtI7DyGJwUnAOywPSvf2OQ,https://instagram.com/mobshitsavage,,43300,0,0,0,0.0,cached,True,21b57a2b-9300-45b8-99bb-b970cb229f22,False,,0.63,Medium,0.8711584148945921,0.0,0.0,0.0,0.0,0.2613475244683776,18.4,7
+1cd4f7d2b1b902a677d0d6fba4096240,$'Edout,https://www.youtube.com/channel/UC6Sj8g3C4Qb9I3bl9xcQDXQ,https://instagram.com/Kid_soja,,21,12683,0,0,0.0,cached,False,,False,,0.28,Low / Emerging,0.2522297719906333,0.5184441001135359,0.0,0.0,0.0,0.20527995662557397,8.7,8
+33f12aeb1989989480885e6a4ed1637f,$ Eli,https://www.youtube.com/channel/UClmpQYORihn-nTO77x3XIpg,https://instagram.com/dollarsigneli,,0,0,0,0,0.0,not_found,False,,False,,0.0,Low / Emerging,0.0,0.0,0.0,0.0,0.0,0.0,0.0,9
+73502f922e4bfbad14f321aa0415f301,$hrames,https://www.youtube.com/channel/UCfwF7qNJylFoSIMi7KTKPdA,https://instagram.com/iPissGreatness,,0,0,0,0,0.0,cached,True,61c33b23-37a9-43a3-9ccd-9b12944ded5b,False,,0.55,Medium,0.0,0.0,0.0,0.0,0.0,0.0,0.0,10

main.py

@@ -0,0 +1,91 @@
+import os
+import argparse
+import pandas as pd
+from dotenv import load_dotenv
+
+from collectors.youtube_collector import YouTubeCollector
+from verifiers.musicbrainz_verifier import MusicBrainzVerifier
+from verifiers.wikidata_verifier import WikidataVerifier
+from core.confidence_scorer import ConfidenceScorer
+from core.ranking_engine import RankingEngine
+
+def run_pipeline(input_csv: str = "cleaned_artists_inventory.csv", limit: int = 10, output_csv: str = "enriched_artists.csv"):
+    load_dotenv()
+    api_key = os.getenv("YOUTUBE_API_KEY")
+
+    if not api_key:
+        print("[Notice] No YOUTUBE_API_KEY found in .env. Running YouTube in offline fallback mode.")
+
+    print(f"Loading data from {input_csv}...")
+    df = pd.read_csv(input_csv)
+    
+    # Process only the selected sample batch limit
+    batch_df = df.head(limit).copy()
+    print(f"Processing sample batch of {len(batch_df)} artists...\n")
+
+    # Initialize all services
+    yt_collector = YouTubeCollector(api_keys=[api_key] if api_key else None)
+    mb_verifier = MusicBrainzVerifier()
+    wd_verifier = WikidataVerifier()
+    scorer = ConfidenceScorer()
+    ranking_engine = RankingEngine()
+
+    results = []
+
+    for idx, row in batch_df.iterrows():
+        name = str(row.get("Name", "")).strip()
+        yt_url = str(row.get("Youtube", "")).strip()
+        print(f"[{idx + 1}/{len(batch_df)}] Analyzing Artist: '{name}'")
+
+        # 1. Collect YouTube 5 metrics
+        yt_data = yt_collector.get_channel_metrics(yt_url, fetch_recent_videos=False)
+
+        # 2. Verify against MusicBrainz
+        mb_data = mb_verifier.verify_artist(name)
+
+        # 3. Verify against Wikidata
+        wd_data = wd_verifier.verify_artist(name)
+
+        # 4. Compute composite confidence score
+        score_data = scorer.calculate_score(yt_data, mb_data, wd_data)
+
+        # Merge results into record
+        results.append({
+            "UID": row.get("UID", ""),
+            "Name": name,
+            "Youtube": yt_url,
+            "Instagram": row.get("Instagram", ""),
+            "Location": row.get("Location", ""),
+            "yt_subscribers": yt_data.get("subscribers", 0),
+            "yt_lifetime_views": yt_data.get("lifetime_views", 0),
+            "yt_video_count": yt_data.get("video_count", 0),
+            "yt_recent_avg_views": yt_data.get("recent_avg_views", 0),
+            "yt_momentum_30d": yt_data.get("momentum_30d", 0.0),
+            "yt_status": yt_data.get("status", ""),
+            "mb_verified": mb_data.get("verified", False),
+            "mb_id": mb_data.get("mbid", ""),
+            "wd_verified": wd_data.get("verified", False),
+            "wd_id": wd_data.get("wikidata_id", ""),
+            "composite_confidence": score_data.get("composite_confidence", 0.0),
+            "confidence_tier": score_data.get("confidence_tier", "Low")
+        })
+
+    raw_out_df = pd.DataFrame(results)
+
+    # 5. Apply Log-Transformation + Min-Max Scaling Ranking Engine
+    print("\nApplying logarithmic normalization & Min-Max ranking...")
+    out_df = ranking_engine.normalize_metrics(raw_out_df)
+
+    out_df.to_csv(output_csv, index=False)
+    print(f"Pipeline run complete. Results saved to '{output_csv}'.\n")
+    
+    # Display the final ranked overview table
+    summary_cols = ["hiffi_rank", "Name", "hiffi_score", "yt_subscribers", "composite_confidence", "confidence_tier"]
+    print(out_df[summary_cols].to_string(index=False))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Pipeline")
+    parser.add_argument("--limit", type=int, default=10, help="Number of records to test")
+    args = parser.parse_args()
+
+    run_pipeline(limit=args.limit)

verifiers/musicbrainz_verifier.py

@@ -0,0 +1,109 @@
+import os
+import json
+import time
+import sqlite3
+import requests
+from typing import Dict, Any
+from rapidfuzz import fuzz
+
+class MusicBrainzVerifier:
+    """
+    Cross-references artist names against MusicBrainz with local SQLite caching.
+    Free, no API key required (Rate limit: 1 request/second).
+    """
+    BASE_URL = "https://musicbrainz.org/ws/2/artist"
+
+    def __init__(self, db_path: str = ".cache/verifiers_cache.sqlite", app_name: str = "HiffiArtistIntelligence", version: str = "1.0"):
+        self.headers = {
+            "User-Agent": f"{app_name}/{version} ( contact@superlabs.co )"
+        }
+        self.last_call_time = 0.0
+        self.db_path = db_path
+        os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
+        self._init_db()
+
+    def _init_db(self):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS musicbrainz_cache (
+                    artist_name TEXT PRIMARY KEY,
+                    data_json TEXT,
+                    cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """)
+
+    def _get_cache(self, name: str) -> Dict[str, Any]:
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT data_json FROM musicbrainz_cache WHERE artist_name = ?", (name.lower(),))
+            row = cursor.fetchone()
+            if row:
+                return json.loads(row[0])
+        return None
+
+    def _set_cache(self, name: str, data: Dict[str, Any]):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO musicbrainz_cache (artist_name, data_json) VALUES (?, ?)",
+                (name.lower(), json.dumps(data))
+            )
+
+    def _rate_limit(self):
+        elapsed = time.time() - self.last_call_time
+        if elapsed < 1.0:
+            time.sleep(1.0 - elapsed)
+        self.last_call_time = time.time()
+
+    def verify_artist(self, name: str) -> Dict[str, Any]:
+        if not name or not name.strip():
+            return {"verified": False, "confidence": 0.0, "reason": "empty_name"}
+
+        cleaned_name = name.strip()
+
+        # Check local cache first
+        cached = self._get_cache(cleaned_name)
+        if cached:
+            return cached
+
+        self._rate_limit()
+        query = f'artist:"{cleaned_name}"'
+        params = {"query": query, "fmt": "json", "limit": 3}
+
+        try:
+            resp = requests.get(self.BASE_URL, headers=self.headers, params=params, timeout=10)
+            if resp.status_code != 200:
+                result = {"verified": False, "confidence": 0.0, "status_code": resp.status_code}
+                return result
+
+            data = resp.json()
+            artists = data.get("artists", [])
+            if not artists:
+                result = {"verified": False, "confidence": 0.0, "reason": "not_found"}
+                self._set_cache(cleaned_name, result)
+                return result
+
+            best_match = artists[0]
+            mb_name = best_match.get("name", "")
+            match_score = fuzz.token_sort_ratio(cleaned_name.lower(), mb_name.lower())
+
+            confidence = round(match_score / 100.0, 2)
+            is_verified = confidence >= 0.80
+
+            result = {
+                "verified": is_verified,
+                "confidence": confidence,
+                "mbid": best_match.get("id"),
+                "matched_name": mb_name,
+                "disambiguation": best_match.get("disambiguation", ""),
+                "country": best_match.get("country", ""),
+                "type": best_match.get("type", "")
+            }
+            self._set_cache(cleaned_name, result)
+            return result
+        except Exception as e:
+            return {"verified": False, "confidence": 0.0, "error": str(e)}
+
+if __name__ == "__main__":
+    verifier = MusicBrainzVerifier()
+    print("Testing MusicBrainz Caching...")
+    print(verifier.verify_artist("Kendrick Lamar"))

verifiers/wikidata_verifier.py

@@ -0,0 +1,105 @@
+import os
+import json
+import sqlite3
+import requests
+from typing import Dict, Any
+from rapidfuzz import fuzz
+
+class WikidataVerifier:
+    """
+    Checks if an artist has a Wikidata entity with local SQLite caching.
+    """
+    SEARCH_URL = "https://www.wikidata.org/w/api.php"
+
+    def __init__(self, db_path: str = ".cache/verifiers_cache.sqlite"):
+        self.headers = {
+            "User-Agent": "HiffiArtistIntelligence/1.0 (contact@superlabs.co)"
+        }
+        self.db_path = db_path
+        os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
+        self._init_db()
+
+    def _init_db(self):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS wikidata_cache (
+                    artist_name TEXT PRIMARY KEY,
+                    data_json TEXT,
+                    cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """)
+
+    def _get_cache(self, name: str) -> Dict[str, Any]:
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT data_json FROM wikidata_cache WHERE artist_name = ?", (name.lower(),))
+            row = cursor.fetchone()
+            if row:
+                return json.loads(row[0])
+        return None
+
+    def _set_cache(self, name: str, data: Dict[str, Any]):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO wikidata_cache (artist_name, data_json) VALUES (?, ?)",
+                (name.lower(), json.dumps(data))
+            )
+
+    def verify_artist(self, name: str) -> Dict[str, Any]:
+        if not name or not name.strip():
+            return {"verified": False, "confidence": 0.0, "reason": "empty_name"}
+
+        cleaned_name = name.strip()
+
+        # Check local cache first
+        cached = self._get_cache(cleaned_name)
+        if cached:
+            return cached
+
+        params = {
+            "action": "wbsearchentities",
+            "format": "json",
+            "language": "en",
+            "type": "item",
+            "search": cleaned_name,
+            "limit": 3
+        }
+
+        try:
+            resp = requests.get(self.SEARCH_URL, headers=self.headers, params=params, timeout=10)
+            if resp.status_code != 200:
+                result = {"verified": False, "confidence": 0.0, "status_code": resp.status_code}
+                return result
+
+            results = resp.json().get("search", [])
+            if not results:
+                result = {"verified": False, "confidence": 0.0, "reason": "not_found"}
+                self._set_cache(cleaned_name, result)
+                return result
+
+            best = results[0]
+            label = best.get("label", "")
+            desc = best.get("description", "").lower()
+            
+            music_keywords = ["musician", "singer", "rapper", "band", "record producer", "songwriter", "artist", "group"]
+            is_music_entity = any(k in desc for k in music_keywords)
+
+            similarity = fuzz.token_sort_ratio(cleaned_name.lower(), label.lower()) / 100.0
+            confidence = round(similarity * (1.0 if is_music_entity else 0.5), 2)
+
+            result = {
+                "verified": confidence >= 0.75,
+                "confidence": confidence,
+                "wikidata_id": best.get("id"),
+                "label": label,
+                "description": best.get("description", "")
+            }
+            self._set_cache(cleaned_name, result)
+            return result
+        except Exception as e:
+            return {"verified": False, "confidence": 0.0, "error": str(e)}
+
+if __name__ == "__main__":
+    verifier = WikidataVerifier()
+    print("Testing Wikidata Caching...")
+    print(verifier.verify_artist("Kendrick Lamar"))