380 lines
15 KiB
Python
380 lines
15 KiB
Python
"""
|
|
test_multiplatform_ranking.py
|
|
Comprehensive QA & Data Governance Test Suite for HIFFI Multi-Platform Pipeline.
|
|
Enforces zero-imputation policy, score bounding, identity resolution,
|
|
fallbacks, strict null retention, and publication eligibility gates.
|
|
"""
|
|
|
|
import inspect
|
|
from pathlib import Path
|
|
import sys
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from core.ranking_engine import HIFFIRankingEngine
|
|
|
|
DATASET_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def dataset():
|
|
if not DATASET_PATH.exists():
|
|
pytest.fail(f"Cohort dataset not found at {DATASET_PATH}")
|
|
return pd.read_csv(DATASET_PATH)
|
|
|
|
|
|
def test_cohort_size(dataset):
|
|
"""Ensures exact 500-artist cohort size."""
|
|
assert len(dataset) == 500, f"Expected 500 artists, found {len(dataset)}"
|
|
|
|
|
|
def test_spotify_streaming_governance(dataset):
|
|
"""Zero-Imputation Policy: spotify_monthly_listeners must not contain imputed zeroes."""
|
|
if "spotify_monthly_listeners" in dataset.columns:
|
|
zero_count = (dataset["spotify_monthly_listeners"] == 0).sum()
|
|
assert zero_count == 0, f"Found {zero_count} imputed 0 values in spotify_monthly_listeners."
|
|
|
|
|
|
def test_missing_metrics_remain_nan(dataset):
|
|
"""Missing metrics must strictly be preserved as NaN."""
|
|
for col in ["spotify_monthly_listeners", "instagram_followers"]:
|
|
if col in dataset.columns:
|
|
assert dataset[col].isna().sum() > 0, f"Expected missing values in {col} to be NaN."
|
|
|
|
|
|
def test_production_ranking_preserved(dataset):
|
|
"""Production ranking baseline must remain unmodified."""
|
|
assert "production_hiffi_rank" in dataset.columns
|
|
assert "production_hiffi_score" in dataset.columns
|
|
valid_prod = dataset["production_hiffi_score"].dropna()
|
|
assert (valid_prod >= 0.0).all() and (valid_prod <= 100.0).all()
|
|
|
|
|
|
def test_score_normalization_bounds(dataset):
|
|
"""HIFFI-XP composite scores must be strictly bounded in [0.0, 100.0]."""
|
|
if "hiffi_xp_score" in dataset.columns:
|
|
valid_scores = dataset["hiffi_xp_score"].dropna()
|
|
assert (valid_scores >= 0.0).all() and (valid_scores <= 100.0).all(), (
|
|
f"XP Scores outside [0, 100] bounds: min={valid_scores.min()}, max={valid_scores.max()}"
|
|
)
|
|
|
|
|
|
def test_rank_monotonicity(dataset):
|
|
"""Higher HIFFI-XP score must yield better (numerically lower) rank."""
|
|
if "hiffi_xp_score" in dataset.columns and "hiffi_xp_rank" in dataset.columns:
|
|
sorted_df = dataset.dropna(subset=["hiffi_xp_score", "hiffi_xp_rank"]).sort_values(by="hiffi_xp_rank")
|
|
scores = sorted_df["hiffi_xp_score"].to_numpy()
|
|
assert (np.diff(scores) <= 1e-5).all(), "Rank monotonicity violated."
|
|
|
|
|
|
def test_schema_provenance_columns(dataset):
|
|
"""All required audit and provenance columns must exist in schema."""
|
|
required = [
|
|
"name", "production_hiffi_score", "production_hiffi_rank",
|
|
"spotify_monthly_listeners", "spotify_daily_change",
|
|
"instagram_followers", "instagram_verification_status",
|
|
"hiffi_xp_score", "hiffi_xp_rank", "rank_delta"
|
|
]
|
|
for col in required:
|
|
assert col in dataset.columns, f"Missing required column: {col}"
|
|
|
|
|
|
def _invoke_instagram_verifier(verifier, artist_name, candidate_handle, profile_data):
|
|
"""Dynamically adapts arguments to match evaluate_candidate or verify_candidate signature."""
|
|
method = None
|
|
for attr in ["evaluate_candidate", "verify_candidate", "verify_artist", "verify"]:
|
|
if hasattr(verifier, attr):
|
|
method = getattr(verifier, attr)
|
|
break
|
|
|
|
if method is None:
|
|
raise AttributeError("Could not find candidate verification method on verifier.")
|
|
|
|
sig = inspect.signature(method)
|
|
params = sig.parameters
|
|
|
|
kwargs = {}
|
|
for p in ["artist_name", "artist", "name"]:
|
|
if p in params:
|
|
kwargs[p] = artist_name
|
|
break
|
|
|
|
for p in ["candidate_handle", "handle", "instagram_handle"]:
|
|
if p in params:
|
|
kwargs[p] = candidate_handle
|
|
break
|
|
|
|
for p in ["profile_data", "profile", "data"]:
|
|
if p in params:
|
|
kwargs[p] = profile_data
|
|
break
|
|
|
|
for p in ["follower_count", "followers"]:
|
|
if p in params:
|
|
kwargs[p] = profile_data.get("followers", 0)
|
|
break
|
|
|
|
for p in ["bio", "biography", "candidate_bio"]:
|
|
if p in params:
|
|
kwargs[p] = profile_data.get("biography", "")
|
|
break
|
|
|
|
for p in ["is_verified", "verified"]:
|
|
if p in params:
|
|
kwargs[p] = profile_data.get("is_verified", False)
|
|
break
|
|
|
|
return method(**kwargs)
|
|
|
|
|
|
def test_instagram_verifier_candidate_matching():
|
|
"""Test algorithmic confidence score calculation for Instagram handle matching."""
|
|
from collectors.instagram_verifier import InstagramCandidateVerifier
|
|
|
|
verifier = InstagramCandidateVerifier()
|
|
res = _invoke_instagram_verifier(
|
|
verifier,
|
|
artist_name="Kendrick Lamar",
|
|
candidate_handle="kendricklamar",
|
|
profile_data={"followers": 15000000, "biography": "pgLang", "full_name": "Kendrick Lamar"}
|
|
)
|
|
status = res.get("status") or res.get("verification_status") or res.get("instagram_verification_status")
|
|
conf = res.get("confidence_score") or res.get("instagram_confidence_score", 0.0)
|
|
evidence = str(res.get("evidence") or res.get("audit_evidence") or res.get("instagram_audit_evidence", ""))
|
|
|
|
assert status == "AUTOMATED_VERIFIED"
|
|
assert conf >= 0.70
|
|
assert "kendricklamar" in evidence.lower() or "exact" in evidence.lower() or len(evidence) > 0
|
|
|
|
|
|
def test_instagram_verifier_suspicious_follower_count():
|
|
"""Ensure handles with fewer than 10,000 followers are quarantined."""
|
|
from collectors.instagram_verifier import InstagramCandidateVerifier
|
|
|
|
verifier = InstagramCandidateVerifier()
|
|
res = _invoke_instagram_verifier(
|
|
verifier,
|
|
artist_name="Drake",
|
|
candidate_handle="drake",
|
|
profile_data={"followers": 2500, "biography": "Fan club", "full_name": "Aubrey Drake"}
|
|
)
|
|
status = res.get("status") or res.get("verification_status") or res.get("instagram_verification_status")
|
|
followers = res.get("followers") or res.get("instagram_followers")
|
|
|
|
assert status in ("UNCERTAIN_NEEDS_REVIEW", "REJECTED_FAN_PAGE", "QUARANTINE_LOW_FOLLOWERS")
|
|
assert followers is None or status != "AUTOMATED_VERIFIED"
|
|
|
|
|
|
def test_spotify_fallback_null_safety():
|
|
"""Verify Spotify embed fallback safely returns structured null dictionary on unindexed items."""
|
|
from collectors.spotify_fallback_collector import SpotifyFallbackCollector
|
|
|
|
fallback = SpotifyFallbackCollector()
|
|
res = fallback.fetch_artist_fallback("Unknown Artist", "spotify:artist:invalid_or_missing_id_999999")
|
|
assert res["spotify_monthly_listeners"] is None
|
|
assert res["spotify_daily_change"] is None
|
|
assert "UNAVAILABLE" in str(res["spotify_status"]) or "EMBED" in str(res["spotify_status"])
|
|
|
|
|
|
def test_spotify_fallback_mocked_network_error(monkeypatch):
|
|
"""Verify Spotify fallback circuit breaker / exception handling gracefully returns null values."""
|
|
from collectors.spotify_fallback_collector import SpotifyFallbackCollector
|
|
import requests
|
|
|
|
def mock_get(*args, **kwargs):
|
|
raise requests.exceptions.ConnectionError("Mocked network failure")
|
|
|
|
monkeypatch.setattr(requests.Session, "get", mock_get)
|
|
|
|
fallback = SpotifyFallbackCollector()
|
|
res = fallback.fetch_artist_fallback("Kendrick Lamar", "spotify:artist:2YZyLo08N0Wb9xBt1NhZWg")
|
|
assert res["spotify_monthly_listeners"] is None
|
|
assert res["spotify_daily_change"] is None
|
|
|
|
|
|
def test_duplicate_records_handled_safely():
|
|
"""Ranking engine handles duplicate artist monikers without crashing."""
|
|
engine = HIFFIRankingEngine()
|
|
test_df = pd.DataFrame([
|
|
{"name": "Baby", "production_hiffi_score": 75.0, "production_hiffi_rank": 10, "spotify_monthly_listeners": 15000000, "spotify_daily_change": 10000, "instagram_followers": 5000000},
|
|
{"name": "Baby", "production_hiffi_score": 40.0, "production_hiffi_rank": 150, "spotify_monthly_listeners": 200000, "spotify_daily_change": 100, "instagram_followers": 50000}
|
|
])
|
|
scored = engine.calculate_hiffi_xp(test_df)
|
|
assert len(scored) == 2
|
|
assert scored.loc[0, "hiffi_xp_score"] > scored.loc[1, "hiffi_xp_score"]
|
|
|
|
|
|
def test_spotify_api_collector_unconfigured(monkeypatch):
|
|
"""Verify that unconfigured credentials safely return null values under zero-imputation."""
|
|
from collectors.spotify_api_collector import SpotifyAPICollector
|
|
|
|
monkeypatch.delenv("SPOTIFY_CLIENT_ID", raising=False)
|
|
monkeypatch.delenv("SPOTIFY_CLIENT_SECRET", raising=False)
|
|
collector = SpotifyAPICollector(client_id="", client_secret="")
|
|
res = collector.fetch_artist("test_id")
|
|
assert res["spotify_status"] == "API_NOT_CONFIGURED"
|
|
assert res["spotify_followers"] is None
|
|
assert res["spotify_popularity"] is None
|
|
|
|
|
|
def test_spotify_api_collector_empty_id():
|
|
"""Verify that empty or invalid artist IDs safely yield None without throwing errors."""
|
|
from collectors.spotify_api_collector import SpotifyAPICollector
|
|
|
|
collector = SpotifyAPICollector(client_id="dummy", client_secret="dummy")
|
|
res = collector.fetch_artist("")
|
|
assert res["spotify_followers"] is None
|
|
assert res["spotify_status"] in ("INVALID_ID", "API_NOT_CONFIGURED")
|
|
|
|
|
|
def test_dynamic_weight_redistribution_fairness():
|
|
"""Ensure artists with missing platforms receive fair proportional scores bounded in [0, 100]."""
|
|
engine = HIFFIRankingEngine()
|
|
test_df = pd.DataFrame([
|
|
{
|
|
"name": "Artist All Signals",
|
|
"production_hiffi_score": 80.0,
|
|
"production_hiffi_rank": 10,
|
|
"spotify_monthly_listeners": 20_000_000,
|
|
"spotify_daily_change": 50_000,
|
|
"instagram_followers": 5_000_000,
|
|
},
|
|
{
|
|
"name": "Artist Missing IG",
|
|
"production_hiffi_score": 80.0,
|
|
"production_hiffi_rank": 10,
|
|
"spotify_monthly_listeners": 20_000_000,
|
|
"spotify_daily_change": 50_000,
|
|
"instagram_followers": np.nan,
|
|
},
|
|
{
|
|
"name": "Artist YouTube Only",
|
|
"production_hiffi_score": 80.0,
|
|
"production_hiffi_rank": 10,
|
|
"spotify_monthly_listeners": np.nan,
|
|
"spotify_daily_change": np.nan,
|
|
"instagram_followers": np.nan,
|
|
}
|
|
])
|
|
|
|
ranked = engine.calculate_hiffi_xp(test_df)
|
|
assert ranked["hiffi_xp_score"].notna().all(), "Redistribution produced unexpected NaN scores"
|
|
assert (ranked["hiffi_xp_score"] >= 0.0).all() and (ranked["hiffi_xp_score"] <= 100.0).all()
|
|
assert ranked.loc[1, "hiffi_xp_score"] > 0.0
|
|
|
|
|
|
def test_score_dominance(dataset):
|
|
"""If Artist A >= Artist B across all present metrics, Score(A) must be >= Score(B)."""
|
|
clean_sub = dataset.dropna(subset=[
|
|
"production_hiffi_score",
|
|
"spotify_monthly_listeners",
|
|
"instagram_followers",
|
|
"hiffi_xp_score"
|
|
]).copy()
|
|
|
|
if len(clean_sub) < 5:
|
|
pytest.skip("Not enough fully populated records to run dominance sample.")
|
|
|
|
sample = clean_sub.head(25).to_dict("records")
|
|
for a in sample:
|
|
for b in sample:
|
|
if (
|
|
a["production_hiffi_score"] >= b["production_hiffi_score"] and
|
|
a["spotify_monthly_listeners"] >= b["spotify_monthly_listeners"] and
|
|
a["instagram_followers"] >= b["instagram_followers"]
|
|
):
|
|
assert a["hiffi_xp_score"] >= (b["hiffi_xp_score"] - 1e-4), (
|
|
f"Dominance violated: {a['name']} ({a['hiffi_xp_score']}) < {b['name']} ({b['hiffi_xp_score']})"
|
|
)
|
|
|
|
|
|
def test_kendall_tau_diagnostic(dataset):
|
|
"""
|
|
Evaluates rank correlation between baseline and composite rank as a diagnostic.
|
|
"""
|
|
from scipy.stats import kendalltau
|
|
|
|
valid = dataset.dropna(subset=["production_hiffi_rank", "hiffi_xp_rank"]).copy()
|
|
if len(valid) < 10:
|
|
pytest.skip("Insufficient ranked rows for Kendall Tau calculation.")
|
|
|
|
tau, p_val = kendalltau(valid["production_hiffi_rank"], valid["hiffi_xp_rank"])
|
|
assert tau > 0.10, f"Rank correlation abnormally low or inverted: tau={tau:.3f}"
|
|
|
|
|
|
def test_score_quantile_spread(dataset):
|
|
"""Ensures composite scores do not clump into a single degenerate value."""
|
|
if "hiffi_xp_score" in dataset.columns:
|
|
valid_scores = dataset["hiffi_xp_score"].dropna()
|
|
if len(valid_scores) > 10:
|
|
assert valid_scores.std() > 5.0, f"Scores exhibit degenerate clumping: std={valid_scores.std():.2f}"
|
|
|
|
|
|
def test_raw_dataset_strict_nan_retention(dataset):
|
|
"""
|
|
Verifies that unindexed secondary fields preserve genuine nulls
|
|
and guards against synthetic zeroes or string-serialized nulls ('nan', 'none', 'null').
|
|
"""
|
|
secondary_metric_cols = [
|
|
"spotify_monthly_listeners",
|
|
"spotify_daily_change",
|
|
"instagram_followers"
|
|
]
|
|
|
|
for col in secondary_metric_cols:
|
|
if col in dataset.columns:
|
|
series = dataset[col]
|
|
# 1. Ensure genuine NaN values exist for unindexed artists
|
|
assert series.isna().any(), f"Column '{col}' contains zero null values across cohort."
|
|
|
|
# 2. Guard against string serialization traps ('nan', 'None', 'null')
|
|
string_nulls = series.dropna().astype(str).str.strip().str.lower().isin(
|
|
["nan", "none", "null", "undefined", "n/a"]
|
|
)
|
|
assert not string_nulls.any(), f"Column '{col}' contains string-serialized null tokens."
|
|
|
|
# 3. Guard against numerical zero substitution
|
|
numeric_vals = pd.to_numeric(series.dropna(), errors="coerce")
|
|
if col in ["spotify_monthly_listeners", "instagram_followers"]:
|
|
zero_records = (numeric_vals == 0).sum()
|
|
assert zero_records == 0, (
|
|
f"Column '{col}' has {zero_records} entries with literal zero (0). "
|
|
"Unindexed platforms must remain NaN."
|
|
)
|
|
|
|
|
|
def test_publication_eligibility_gates(dataset):
|
|
"""Verifies that publication gates enforce evidence tiers without altering raw scores."""
|
|
engine = HIFFIRankingEngine()
|
|
gated = engine.apply_publication_eligibility_gates(dataset)
|
|
|
|
# 1. Raw scores must remain unchanged
|
|
assert (gated["hiffi_xp_score"] == dataset["hiffi_xp_score"]).all(), (
|
|
"Publication gates altered the underlying raw composite scores."
|
|
)
|
|
|
|
# 2. Check Top 10 requires all 3 platforms
|
|
top_10 = gated[gated["hiffi_xp_published_rank"] <= 10]
|
|
assert len(top_10) == 10
|
|
assert (top_10["platform_evidence_tier"] == 3).all(), (
|
|
f"Found artists with < 3 platforms in Top 10 published ranks: "
|
|
f"{top_10[top_10['platform_evidence_tier'] < 3][['name', 'platform_evidence_tier']]}"
|
|
)
|
|
|
|
# 3. Check Top 50 requires at least 2 platforms
|
|
top_50 = gated[gated["hiffi_xp_published_rank"] <= 50]
|
|
assert len(top_50) == 50
|
|
assert (top_50["platform_evidence_tier"] >= 2).all(), (
|
|
f"Found single-platform artists in Top 50 published ranks: "
|
|
f"{top_50[top_50['platform_evidence_tier'] < 2][['name', 'platform_evidence_tier']]}"
|
|
)
|
|
|
|
# 4. Check ranks are strictly unique and consecutive 1 to N
|
|
ranks = gated["hiffi_xp_published_rank"].sort_values().tolist()
|
|
expected_ranks = list(range(1, len(gated) + 1))
|
|
assert ranks == expected_ranks, "Published ranks contain gaps or duplicates." |