checkpoint: working 10-artist batch with log-normalized ranking

core/confidence_scorer.py

@@ -0,0 +1,77 @@
+from typing import Dict, Any
+
+class ConfidenceScorer:
+    """
+    Computes a weighted authenticity and quality score across multiple data sources:
+    - YouTube footprint: 40%
+    - MusicBrainz catalog: 35%
+    - Wikidata entity: 25%
+    """
+    def __init__(self, yt_weight: float = 0.40, mb_weight: float = 0.35, wd_weight: float = 0.25):
+        self.w_yt = yt_weight
+        self.w_mb = mb_weight
+        self.w_wd = wd_weight
+
+    def calculate_score(
+        self,
+        yt_metrics: Dict[str, Any],
+        mb_result: Dict[str, Any],
+        wd_result: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        # 1. YouTube sub-score (0.0 to 1.0)
+        yt_score = 0.0
+        if yt_metrics.get("status") in ["live_api", "cached"]:
+            subs = yt_metrics.get("subscribers", 0)
+            views = yt_metrics.get("lifetime_views", 0)
+            vids = yt_metrics.get("video_count", 0)
+
+            # Baseline 0.5 for a verified channel existence
+            yt_score = 0.5
+            if subs >= 1000:
+                yt_score += 0.2
+            elif subs > 0:
+                yt_score += 0.1
+
+            if views >= 50000:
+                yt_score += 0.2
+            elif views > 0:
+                yt_score += 0.1
+
+            if vids >= 5:
+                yt_score += 0.1
+
+        # 2. MusicBrainz sub-score
+        mb_score = mb_result.get("confidence", 0.0) if mb_result.get("verified") else 0.0
+
+        # 3. Wikidata sub-score
+        wd_score = wd_result.get("confidence", 0.0) if wd_result.get("verified") else 0.0
+
+        # Aggregated composite score
+        composite = (yt_score * self.w_yt) + (mb_score * self.w_mb) + (wd_score * self.w_wd)
+        composite = round(min(1.0, max(0.0, composite)), 2)
+
+        # Confidence category
+        if composite >= 0.75:
+            tier = "High"
+        elif composite >= 0.45:
+            tier = "Medium"
+        else:
+            tier = "Low / Emerging"
+
+        return {
+            "composite_confidence": composite,
+            "confidence_tier": tier,
+            "sub_scores": {
+                "youtube": round(yt_score, 2),
+                "musicbrainz": round(mb_score, 2),
+                "wikidata": round(wd_score, 2)
+            }
+        }
+
+if __name__ == "__main__":
+    scorer = ConfidenceScorer()
+    dummy_yt = {"status": "live_api", "subscribers": 5000, "lifetime_views": 250000, "video_count": 18}
+    dummy_mb = {"verified": True, "confidence": 1.0}
+    dummy_wd = {"verified": True, "confidence": 1.0}
+    print("Test Scorer Output:")
+    print(scorer.calculate_score(dummy_yt, dummy_mb, dummy_wd))

core/deduplication.py

@@ -0,0 +1,59 @@
+import re
+import pandas as pd
+from urllib.parse import urlparse
+
+def clean_url(url: str) -> str:
+    """Normalize URLs by stripping query params, trailing slashes, and spaces."""
+    if not isinstance(url, str) or not url.strip() or url.strip().lower() in ["nan", "none", ""]:
+        return ""
+    url = url.strip().rstrip("/")
+    if not re.match(r"^https?://", url, re.I):
+        url = "https://" + url
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}{parsed.path}".rstrip("/")
+
+def extract_channel_key(yt_url: str) -> str:
+    """Extract unique channel ID or handle to catch cross-name channel duplicates."""
+    clean = clean_url(yt_url).lower()
+    if not clean:
+        return ""
+    parts = [p for p in urlparse(clean).path.split("/") if p]
+    if not parts:
+        return ""
+    return parts[-1]
+
+def deduplicate_inventory(input_path: str = "artist_inventory_artists.csv") -> pd.DataFrame:
+    print(f"Loading {input_path}...")
+    df = pd.read_csv(input_path)
+    initial_count = len(df)
+    print(f"Initial raw rows: {initial_count}")
+
+    # 1. Clean strings and URLs
+    for col in ["Name", "Location"]:
+        df[col] = df[col].fillna("").astype(str).str.strip()
+
+    for col in ["Instagram", "Youtube", "TikTok", "Facebook"]:
+        df[col] = df[col].apply(clean_url)
+
+    # 2. Generate normalized match keys
+    df["clean_name"] = df["Name"].str.lower().str.replace(r"[^\w\s]", "", regex=True).str.strip()
+    df["yt_key"] = df["Youtube"].apply(extract_channel_key)
+
+    # 3. Drop exact duplicates on Name + YouTube
+    df_dedup = df.drop_duplicates(subset=["clean_name", "yt_key"], keep="first").copy()
+    
+    # 4. Check for duplicate YouTube channels across different names
+    yt_conflicts = df_dedup[df_dedup["yt_key"] != ""].duplicated(subset=["yt_key"], keep=False).sum()
+    print(f"Rows with shared YouTube channels across different names: {yt_conflicts}")
+
+    dropped = initial_count - len(df_dedup)
+    print(f"Deduplicated count: {len(df_dedup)} (Dropped {dropped} duplicate rows)")
+
+    # Clean temporary helper columns before returning
+    df_dedup.drop(columns=["clean_name", "yt_key"], inplace=True)
+    return df_dedup
+
+if __name__ == "__main__":
+    cleaned_df = deduplicate_inventory()
+    cleaned_df.to_csv("cleaned_artists_inventory.csv", index=False)
+    print("Saved cleaned data to cleaned_artists_inventory.csv")

core/ranking_engine.py

@@ -0,0 +1,83 @@
+import numpy as np
+import pandas as pd
+from typing import Dict, Any
+
+class RankingEngine:
+    """
+    Computes equitable artist rankings using Log Transformation (log1p)
+    and Min-Max Scaling across skewed social/streaming distribution metrics.
+    """
+    DEFAULT_WEIGHTS = {
+        "norm_subscribers": 0.30,
+        "norm_lifetime_views": 0.25,
+        "norm_recent_avg_views": 0.25,
+        "norm_video_count": 0.10,
+        "norm_momentum_30d": 0.10
+    }
+
+    def __init__(self, weights: Dict[str, float] = None):
+        self.weights = weights or self.DEFAULT_WEIGHTS
+
+    def normalize_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Applies log10(1 + x) and Min-Max scaling to compress power-law tails."""
+        df_scored = df.copy()
+
+        metrics_to_transform = [
+            ("yt_subscribers", "norm_subscribers"),
+            ("yt_lifetime_views", "norm_lifetime_views"),
+            ("yt_video_count", "norm_video_count"),
+            ("yt_recent_avg_views", "norm_recent_avg_views"),
+            ("yt_momentum_30d", "norm_momentum_30d")
+        ]
+
+        for raw_col, norm_col in metrics_to_transform:
+            if raw_col not in df_scored.columns:
+                df_scored[norm_col] = 0.0
+                continue
+
+            # Step 1: Handle NaNs and negative values, then apply Log10(1 + x)
+            series = pd.to_numeric(df_scored[raw_col], errors="coerce").fillna(0).clip(lower=0)
+            log_series = np.log10(1 + series)
+
+            min_val = log_series.min()
+            max_val = log_series.max()
+
+            # Step 2: Min-Max Normalization to scale [0.0, 1.0]
+            if max_val > min_val:
+                df_scored[norm_col] = (log_series - min_val) / (max_val - min_val)
+            else:
+                df_scored[norm_col] = 0.0
+
+        # Step 3: Compute weighted base score
+        df_scored["base_rank_score"] = sum(
+            df_scored[col] * weight for col, weight in self.weights.items()
+        )
+
+        # Step 4: Multiply by verification confidence to penalize unverified or fake profiles
+        # Artists with verified MusicBrainz/Wikidata and legitimate channels get full weight
+        confidence = df_scored.get("composite_confidence", pd.Series(1.0, index=df_scored.index))
+        # Ensure a minimum floor of 0.2 so emerging artists are ranked on data, not dropped completely
+        confidence_multiplier = 0.2 + (0.8 * confidence)
+        
+        # Step 5: Final HIFFI Score on a clean 0 to 100 scale
+        df_scored["hiffi_score"] = (df_scored["base_rank_score"] * confidence_multiplier * 100).round(2)
+        
+        # Sort and assign explicit rank ordinal
+        df_scored = df_scored.sort_values(by="hiffi_score", ascending=False).reset_index(drop=True)
+        df_scored["hiffi_rank"] = df_scored.index + 1
+
+        return df_scored
+
+if __name__ == "__main__":
+    # Test script with dummy mock distribution
+    sample_data = pd.DataFrame([
+        {"Name": "Global Superstar", "yt_subscribers": 15000000, "yt_lifetime_views": 8500000000, "yt_video_count": 120, "yt_recent_avg_views": 2500000, "yt_momentum_30d": 1.2, "composite_confidence": 0.95},
+        {"Name": "Mid-tier Rising", "yt_subscribers": 250000, "yt_lifetime_views": 45000000, "yt_video_count": 45, "yt_recent_avg_views": 180000, "yt_momentum_30d": 5.4, "composite_confidence": 0.85},
+        {"Name": "Emerging Talent", "yt_subscribers": 12000, "yt_lifetime_views": 850000, "yt_video_count": 14, "yt_recent_avg_views": 45000, "yt_momentum_30d": 12.8, "composite_confidence": 0.60},
+        {"Name": "Dormant / Inactive", "yt_subscribers": 500, "yt_lifetime_views": 15000, "yt_video_count": 3, "yt_recent_avg_views": 50, "yt_momentum_30d": 0.0, "composite_confidence": 0.35}
+    ])
+    
+    engine = RankingEngine()
+    ranked = engine.normalize_metrics(sample_data)
+    print("Ranked Test Artists with Log & Min-Max Transformation:")
+    print(ranked[["hiffi_rank", "Name", "hiffi_score", "norm_subscribers", "norm_lifetime_views"]])