fix: resolve governance gaps, patch credential exposure, and connect multi-platform dashboard
This commit is contained in:
Srilekha committed
1 parent
6f2e8e0c9d
commit
7fb7d775d8
5 files changed
+324
-387
No files matched your search
@@ -11,7 +11,6 @@ from pathlib import Path
|
||||
import sys
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -22,6 +21,7 @@ from collectors.instagram_verifier import InstagramCandidateVerifier
|
||||
from collectors.kworb_spotify_collector import KworbStreamingCollector
|
||||
from collectors.spotify_api_collector import SpotifyAPICollector
|
||||
from collectors.spotify_fallback_collector import SpotifyFallbackCollector
|
||||
from core.ranking_engine import HIFFIRankingEngine
|
||||
|
||||
DATA_DIR = PROJECT_ROOT / "data"
|
||||
BASE_DATASET = DATA_DIR / "hiffi_multiplatform_500.csv"
|
||||
@@ -39,70 +39,16 @@ def load_handle_overrides() -> Dict[str, object]:
|
||||
return {}
|
||||
|
||||
|
||||
def _normalize_metric(series: pd.Series) -> pd.Series:
|
||||
"""
|
||||
Direct array-based log-scaling and min-max normalization to [0.0, 1.0].
|
||||
Preserves index alignment and avoids dictionary/map collisions.
|
||||
"""
|
||||
clean_series = pd.to_numeric(series, errors="coerce")
|
||||
out = pd.Series(0.0, index=series.index, dtype=float)
|
||||
valid_mask = clean_series.notna() & (clean_series > 0)
|
||||
|
||||
if not valid_mask.any():
|
||||
return out
|
||||
|
||||
log_vals = np.log1p(clean_series[valid_mask])
|
||||
min_v, max_v = log_vals.min(), log_vals.max()
|
||||
|
||||
if max_v > min_v:
|
||||
out.loc[valid_mask] = (log_vals - min_v) / (max_v - min_v)
|
||||
else:
|
||||
out.loc[valid_mask] = 1.0
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def calculate_hiffi_xp_scores(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Computes experimental multi-platform HIFFI-XP scores and ranks.
|
||||
Strictly preserves production_hiffi_rank and production_hiffi_score.
|
||||
Weights: 50% YouTube production baseline, 30% Spotify telemetry, 20% Instagram reach.
|
||||
Delegates to the canonical HIFFIRankingEngine to enforce zero-imputation,
|
||||
proportional dynamic redistribution, anomaly detection, and publication eligibility gates.
|
||||
"""
|
||||
out = df.copy()
|
||||
|
||||
# Ensure baseline columns exist
|
||||
if "production_hiffi_score" not in out.columns and "hiffi_score" in out.columns:
|
||||
out["production_hiffi_score"] = out["hiffi_score"]
|
||||
if "production_hiffi_rank" not in out.columns and "hiffi_rank" in out.columns:
|
||||
out["production_hiffi_rank"] = out["hiffi_rank"]
|
||||
|
||||
# 1. YouTube production baseline signal (0.0 to 1.0)
|
||||
yt_score = pd.to_numeric(out.get("production_hiffi_score", out.get("hiffi_score", 0)), errors="coerce").fillna(0.0)
|
||||
# If production scores are already on a 0-100 scale, normalize directly
|
||||
if yt_score.max() > 1.0:
|
||||
yt_norm = (yt_score / 100.0).clip(0.0, 1.0)
|
||||
else:
|
||||
yt_norm = _normalize_metric(yt_score)
|
||||
|
||||
# 2. Spotify streaming signal (0.0 to 1.0)
|
||||
sp_norm = _normalize_metric(out.get("spotify_monthly_listeners"))
|
||||
|
||||
# 3. Instagram reach signal (0.0 to 1.0)
|
||||
ig_norm = _normalize_metric(out.get("instagram_followers"))
|
||||
|
||||
# Multi-platform composite weighting: 50% YT + 30% SP + 20% IG
|
||||
xp_composite = (0.50 * yt_norm + 0.30 * sp_norm + 0.20 * ig_norm) * 100.0
|
||||
out["hiffi_xp_score"] = xp_composite.clip(lower=0.0, upper=100.0).round(2)
|
||||
|
||||
# Monotonic ranking (higher score -> lower rank integer)
|
||||
out["hiffi_xp_rank"] = out["hiffi_xp_score"].rank(ascending=False, method="min").astype(int)
|
||||
|
||||
# Calculate rank delta compared to YouTube production baseline
|
||||
if "production_hiffi_rank" in out.columns:
|
||||
prod_rank = pd.to_numeric(out["production_hiffi_rank"], errors="coerce")
|
||||
out["rank_delta"] = prod_rank - out["hiffi_xp_rank"]
|
||||
|
||||
return out
|
||||
engine = HIFFIRankingEngine()
|
||||
scored = engine.calculate_hiffi_xp(df)
|
||||
flagged = engine.detect_metric_anomalies(scored)
|
||||
gated = engine.apply_publication_eligibility_gates(flagged)
|
||||
return gated
|
||||
|
||||
|
||||
def run_multiplatform_pipeline(
|
||||
@@ -125,6 +71,12 @@ def run_multiplatform_pipeline(
|
||||
initial_ig_verified = df["instagram_followers"].notna().sum() if "instagram_followers" in df.columns else 0
|
||||
print(f"[Loader] Loaded {len(df)} checkpoints ({initial_ig_verified} verified follower records).")
|
||||
|
||||
now_utc = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# Ensure baseline YouTube timestamps exist for freshness validation
|
||||
if "youtube_updated_at" not in df.columns or df["youtube_updated_at"].isna().all():
|
||||
df["youtube_updated_at"] = df.get("updated_at", now_utc)
|
||||
|
||||
# 1. Instagram Candidate Verification & Overrides Ingestion
|
||||
print("\n[Phase 1] Executing Automated Instagram Verification & Provenance...")
|
||||
overrides = load_handle_overrides()
|
||||
@@ -137,8 +89,6 @@ def run_multiplatform_pipeline(
|
||||
ig_evidence = []
|
||||
ig_dates = []
|
||||
|
||||
now_utc = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
for _, row in df.iterrows():
|
||||
name = str(row["name"]).strip()
|
||||
override = overrides.get(name)
|
||||
@@ -169,7 +119,7 @@ def run_multiplatform_pipeline(
|
||||
ver_res = verifier.evaluate_candidate(
|
||||
artist_name=name,
|
||||
candidate_handle=cand_handle,
|
||||
follower_count=cand_followers if pd.notna(cand_followers) else 0
|
||||
follower_count=cand_followers if pd.notna(cand_followers) else None
|
||||
)
|
||||
elif hasattr(verifier, "verify_candidate"):
|
||||
ver_res = verifier.verify_candidate(
|
||||
@@ -199,6 +149,7 @@ def run_multiplatform_pipeline(
|
||||
df["instagram_verification_status"] = ig_status
|
||||
df["instagram_audit_evidence"] = ig_evidence
|
||||
df["instagram_collected_at"] = ig_dates
|
||||
df["instagram_updated_at"] = ig_dates
|
||||
|
||||
# 2. Spotify Multi-Tier Telemetry Ingestion (Tier 1: Kworb -> Tier 2: Official API -> Tier 3: Embed Fallback)
|
||||
print("\n[Phase 2] Ingesting Spotify Telemetry (Tier 1: Kworb -> Tier 2: Official API -> Tier 3: Embed Fallback)...")
|
||||
@@ -264,8 +215,9 @@ def run_multiplatform_pipeline(
|
||||
df["spotify_data_source"] = sp_source
|
||||
df["spotify_status"] = sp_status
|
||||
df["spotify_collected_at"] = sp_dates
|
||||
df["spotify_updated_at"] = sp_dates
|
||||
|
||||
# 3. HIFFI-XP Score Normalization & Monotonic Ranking
|
||||
# 3. Canonical Scoring & Governance
|
||||
scored_df = calculate_hiffi_xp_scores(df)
|
||||
|
||||
# Save finalized cohort
|
||||
|
||||
Reference in new issue
Block a user