"""
test_multiplatform_ranking.py
Comprehensive QA & Data Governance Test Suite for HIFFI Multi-Platform Pipeline.
Enforces zero-imputation policy, score bounding, identity resolution,
fallbacks, strict null retention, and publication eligibility gates.
"""

import inspect
from pathlib import Path
import sys
import numpy as np
import pandas as pd
import pytest

PROJECT_ROOT = Path(__file__).resolve().parent
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from core.ranking_engine import HIFFIRankingEngine

DATASET_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"


@pytest.fixture(scope="module")
def dataset():
    if not DATASET_PATH.exists():
        pytest.fail(f"Cohort dataset not found at {DATASET_PATH}")
    return pd.read_csv(DATASET_PATH)


def test_cohort_size(dataset):
    """Ensures exact 500-artist cohort size."""
    assert len(dataset) == 500, f"Expected 500 artists, found {len(dataset)}"


def test_spotify_streaming_governance(dataset):
    """Zero-Imputation Policy: spotify_monthly_listeners must not contain imputed zeroes."""
    if "spotify_monthly_listeners" in dataset.columns:
        zero_count = (dataset["spotify_monthly_listeners"] == 0).sum()
        assert zero_count == 0, f"Found {zero_count} imputed 0 values in spotify_monthly_listeners."


def test_missing_metrics_remain_nan(dataset):
    """Missing metrics must strictly be preserved as NaN."""
    for col in ["spotify_monthly_listeners", "instagram_followers"]:
        if col in dataset.columns:
            assert dataset[col].isna().sum() > 0, f"Expected missing values in {col} to be NaN."


def test_production_ranking_preserved(dataset):
    """Production ranking baseline must remain unmodified."""
    assert "production_hiffi_rank" in dataset.columns
    assert "production_hiffi_score" in dataset.columns
    valid_prod = dataset["production_hiffi_score"].dropna()
    assert (valid_prod >= 0.0).all() and (valid_prod <= 100.0).all()


def test_score_normalization_bounds(dataset):
    """HIFFI-XP composite scores must be strictly bounded in [0.0, 100.0]."""
    if "hiffi_xp_score" in dataset.columns:
        valid_scores = dataset["hiffi_xp_score"].dropna()
        assert (valid_scores >= 0.0).all() and (valid_scores <= 100.0).all(), (
            f"XP Scores outside [0, 100] bounds: min={valid_scores.min()}, max={valid_scores.max()}"
        )


def test_rank_monotonicity(dataset):
    """Higher HIFFI-XP score must yield better (numerically lower) rank."""
    if "hiffi_xp_score" in dataset.columns and "hiffi_xp_rank" in dataset.columns:
        sorted_df = dataset.dropna(subset=["hiffi_xp_score", "hiffi_xp_rank"]).sort_values(by="hiffi_xp_rank")
        scores = sorted_df["hiffi_xp_score"].to_numpy()
        assert (np.diff(scores) <= 1e-5).all(), "Rank monotonicity violated."


def test_schema_provenance_columns(dataset):
    """All required audit and provenance columns must exist in schema."""
    required = [
        "name", "production_hiffi_score", "production_hiffi_rank",
        "spotify_monthly_listeners", "spotify_daily_change",
        "instagram_followers", "instagram_verification_status",
        "hiffi_xp_score", "hiffi_xp_rank", "rank_delta"
    ]
    for col in required:
        assert col in dataset.columns, f"Missing required column: {col}"


def _invoke_instagram_verifier(verifier, artist_name, candidate_handle, profile_data):
    """Dynamically adapts arguments to match evaluate_candidate or verify_candidate signature."""
    method = None
    for attr in ["evaluate_candidate", "verify_candidate", "verify_artist", "verify"]:
        if hasattr(verifier, attr):
            method = getattr(verifier, attr)
            break

    if method is None:
        raise AttributeError("Could not find candidate verification method on verifier.")

    sig = inspect.signature(method)
    params = sig.parameters

    kwargs = {}
    for p in ["artist_name", "artist", "name"]:
        if p in params:
            kwargs[p] = artist_name
            break

    for p in ["candidate_handle", "handle", "instagram_handle"]:
        if p in params:
            kwargs[p] = candidate_handle
            break

    for p in ["profile_data", "profile", "data"]:
        if p in params:
            kwargs[p] = profile_data
            break

    for p in ["follower_count", "followers"]:
        if p in params:
            kwargs[p] = profile_data.get("followers", 0)
            break

    for p in ["bio", "biography", "candidate_bio"]:
        if p in params:
            kwargs[p] = profile_data.get("biography", "")
            break

    for p in ["is_verified", "verified"]:
        if p in params:
            kwargs[p] = profile_data.get("is_verified", False)
            break

    return method(**kwargs)


def test_instagram_verifier_candidate_matching():
    """Test algorithmic confidence score calculation for Instagram handle matching."""
    from collectors.instagram_verifier import InstagramCandidateVerifier

    verifier = InstagramCandidateVerifier()
    res = _invoke_instagram_verifier(
        verifier,
        artist_name="Kendrick Lamar",
        candidate_handle="kendricklamar",
        profile_data={"followers": 15000000, "biography": "pgLang", "full_name": "Kendrick Lamar"}
    )
    status = res.get("status") or res.get("verification_status") or res.get("instagram_verification_status")
    conf = res.get("confidence_score") or res.get("instagram_confidence_score", 0.0)
    evidence = str(res.get("evidence") or res.get("audit_evidence") or res.get("instagram_audit_evidence", ""))

    assert status == "AUTOMATED_VERIFIED"
    assert conf >= 0.70
    assert "kendricklamar" in evidence.lower() or "exact" in evidence.lower() or len(evidence) > 0


def test_instagram_verifier_suspicious_follower_count():
    """Ensure handles with fewer than 10,000 followers are quarantined."""
    from collectors.instagram_verifier import InstagramCandidateVerifier

    verifier = InstagramCandidateVerifier()
    res = _invoke_instagram_verifier(
        verifier,
        artist_name="Drake",
        candidate_handle="drake",
        profile_data={"followers": 2500, "biography": "Fan club", "full_name": "Aubrey Drake"}
    )
    status = res.get("status") or res.get("verification_status") or res.get("instagram_verification_status")
    followers = res.get("followers") or res.get("instagram_followers")

    assert status in ("UNCERTAIN_NEEDS_REVIEW", "REJECTED_FAN_PAGE", "QUARANTINE_LOW_FOLLOWERS")
    assert followers is None or status != "AUTOMATED_VERIFIED"


def test_spotify_fallback_null_safety():
    """Verify Spotify embed fallback safely returns structured null dictionary on unindexed items."""
    from collectors.spotify_fallback_collector import SpotifyFallbackCollector

    fallback = SpotifyFallbackCollector()
    res = fallback.fetch_artist_fallback("Unknown Artist", "spotify:artist:invalid_or_missing_id_999999")
    assert res["spotify_monthly_listeners"] is None
    assert res["spotify_daily_change"] is None
    assert "UNAVAILABLE" in str(res["spotify_status"]) or "EMBED" in str(res["spotify_status"])


def test_spotify_fallback_mocked_network_error(monkeypatch):
    """Verify Spotify fallback circuit breaker / exception handling gracefully returns null values."""
    from collectors.spotify_fallback_collector import SpotifyFallbackCollector
    import requests

    def mock_get(*args, **kwargs):
        raise requests.exceptions.ConnectionError("Mocked network failure")

    monkeypatch.setattr(requests.Session, "get", mock_get)

    fallback = SpotifyFallbackCollector()
    res = fallback.fetch_artist_fallback("Kendrick Lamar", "spotify:artist:2YZyLo08N0Wb9xBt1NhZWg")
    assert res["spotify_monthly_listeners"] is None
    assert res["spotify_daily_change"] is None


def test_duplicate_records_handled_safely():
    """Ranking engine handles duplicate artist monikers without crashing."""
    engine = HIFFIRankingEngine()
    test_df = pd.DataFrame([
        {"name": "Baby", "production_hiffi_score": 75.0, "production_hiffi_rank": 10, "spotify_monthly_listeners": 15000000, "spotify_daily_change": 10000, "instagram_followers": 5000000},
        {"name": "Baby", "production_hiffi_score": 40.0, "production_hiffi_rank": 150, "spotify_monthly_listeners": 200000, "spotify_daily_change": 100, "instagram_followers": 50000}
    ])
    scored = engine.calculate_hiffi_xp(test_df)
    assert len(scored) == 2
    assert scored.loc[0, "hiffi_xp_score"] > scored.loc[1, "hiffi_xp_score"]


def test_spotify_api_collector_unconfigured(monkeypatch):
    """Verify that unconfigured credentials safely return null values under zero-imputation."""
    from collectors.spotify_api_collector import SpotifyAPICollector

    monkeypatch.delenv("SPOTIFY_CLIENT_ID", raising=False)
    monkeypatch.delenv("SPOTIFY_CLIENT_SECRET", raising=False)
    collector = SpotifyAPICollector(client_id="", client_secret="")
    res = collector.fetch_artist("test_id")
    assert res["spotify_status"] == "API_NOT_CONFIGURED"
    assert res["spotify_followers"] is None
    assert res["spotify_popularity"] is None


def test_spotify_api_collector_empty_id():
    """Verify that empty or invalid artist IDs safely yield None without throwing errors."""
    from collectors.spotify_api_collector import SpotifyAPICollector

    collector = SpotifyAPICollector(client_id="dummy", client_secret="dummy")
    res = collector.fetch_artist("")
    assert res["spotify_followers"] is None
    assert res["spotify_status"] in ("INVALID_ID", "API_NOT_CONFIGURED")


def test_dynamic_weight_redistribution_fairness():
    """Ensure artists with missing platforms receive fair proportional scores bounded in [0, 100]."""
    engine = HIFFIRankingEngine()
    test_df = pd.DataFrame([
        {
            "name": "Artist All Signals",
            "production_hiffi_score": 80.0,
            "production_hiffi_rank": 10,
            "spotify_monthly_listeners": 20_000_000,
            "spotify_daily_change": 50_000,
            "instagram_followers": 5_000_000,
        },
        {
            "name": "Artist Missing IG",
            "production_hiffi_score": 80.0,
            "production_hiffi_rank": 10,
            "spotify_monthly_listeners": 20_000_000,
            "spotify_daily_change": 50_000,
            "instagram_followers": np.nan,
        },
        {
            "name": "Artist YouTube Only",
            "production_hiffi_score": 80.0,
            "production_hiffi_rank": 10,
            "spotify_monthly_listeners": np.nan,
            "spotify_daily_change": np.nan,
            "instagram_followers": np.nan,
        }
    ])

    ranked = engine.calculate_hiffi_xp(test_df)
    assert ranked["hiffi_xp_score"].notna().all(), "Redistribution produced unexpected NaN scores"
    assert (ranked["hiffi_xp_score"] >= 0.0).all() and (ranked["hiffi_xp_score"] <= 100.0).all()
    assert ranked.loc[1, "hiffi_xp_score"] > 0.0


def test_score_dominance(dataset):
    """If Artist A >= Artist B across all present metrics, Score(A) must be >= Score(B)."""
    clean_sub = dataset.dropna(subset=[
        "production_hiffi_score",
        "spotify_monthly_listeners",
        "instagram_followers",
        "hiffi_xp_score"
    ]).copy()

    if len(clean_sub) < 5:
        pytest.skip("Not enough fully populated records to run dominance sample.")

    sample = clean_sub.head(25).to_dict("records")
    for a in sample:
        for b in sample:
            if (
                a["production_hiffi_score"] >= b["production_hiffi_score"] and
                a["spotify_monthly_listeners"] >= b["spotify_monthly_listeners"] and
                a["instagram_followers"] >= b["instagram_followers"]
            ):
                assert a["hiffi_xp_score"] >= (b["hiffi_xp_score"] - 1e-4), (
                    f"Dominance violated: {a['name']} ({a['hiffi_xp_score']}) < {b['name']} ({b['hiffi_xp_score']})"
                )


def test_kendall_tau_diagnostic(dataset):
    """
    Evaluates rank correlation between baseline and composite rank as a diagnostic.
    """
    from scipy.stats import kendalltau

    valid = dataset.dropna(subset=["production_hiffi_rank", "hiffi_xp_rank"]).copy()
    if len(valid) < 10:
        pytest.skip("Insufficient ranked rows for Kendall Tau calculation.")

    tau, p_val = kendalltau(valid["production_hiffi_rank"], valid["hiffi_xp_rank"])
    assert tau > 0.10, f"Rank correlation abnormally low or inverted: tau={tau:.3f}"


def test_score_quantile_spread(dataset):
    """Ensures composite scores do not clump into a single degenerate value."""
    if "hiffi_xp_score" in dataset.columns:
        valid_scores = dataset["hiffi_xp_score"].dropna()
        if len(valid_scores) > 10:
            assert valid_scores.std() > 5.0, f"Scores exhibit degenerate clumping: std={valid_scores.std():.2f}"


def test_raw_dataset_strict_nan_retention(dataset):
    """
    Verifies that unindexed secondary fields preserve genuine nulls
    and guards against synthetic zeroes or string-serialized nulls ('nan', 'none', 'null').
    """
    secondary_metric_cols = [
        "spotify_monthly_listeners",
        "spotify_daily_change",
        "instagram_followers"
    ]

    for col in secondary_metric_cols:
        if col in dataset.columns:
            series = dataset[col]
            # 1. Ensure genuine NaN values exist for unindexed artists
            assert series.isna().any(), f"Column '{col}' contains zero null values across cohort."

            # 2. Guard against string serialization traps ('nan', 'None', 'null')
            string_nulls = series.dropna().astype(str).str.strip().str.lower().isin(
                ["nan", "none", "null", "undefined", "n/a"]
            )
            assert not string_nulls.any(), f"Column '{col}' contains string-serialized null tokens."

            # 3. Guard against numerical zero substitution
            numeric_vals = pd.to_numeric(series.dropna(), errors="coerce")
            if col in ["spotify_monthly_listeners", "instagram_followers"]:
                zero_records = (numeric_vals == 0).sum()
                assert zero_records == 0, (
                    f"Column '{col}' has {zero_records} entries with literal zero (0). "
                    "Unindexed platforms must remain NaN."
                )


def test_publication_eligibility_gates(dataset):
    """Verifies that publication gates enforce evidence tiers without altering raw scores."""
    engine = HIFFIRankingEngine()
    gated = engine.apply_publication_eligibility_gates(dataset)

    # 1. Raw scores must remain unchanged
    assert (gated["hiffi_xp_score"] == dataset["hiffi_xp_score"]).all(), (
        "Publication gates altered the underlying raw composite scores."
    )

    # 2. Check Top 10 requires all 3 platforms
    top_10 = gated[gated["hiffi_xp_published_rank"] <= 10]
    assert len(top_10) == 10
    assert (top_10["platform_evidence_tier"] == 3).all(), (
        f"Found artists with < 3 platforms in Top 10 published ranks: "
        f"{top_10[top_10['platform_evidence_tier'] < 3][['name', 'platform_evidence_tier']]}"
    )

    # 3. Check Top 50 requires at least 2 platforms
    top_50 = gated[gated["hiffi_xp_published_rank"] <= 50]
    assert len(top_50) == 50
    assert (top_50["platform_evidence_tier"] >= 2).all(), (
        f"Found single-platform artists in Top 50 published ranks: "
        f"{top_50[top_50['platform_evidence_tier'] < 2][['name', 'platform_evidence_tier']]}"
    )

    # 4. Check ranks are strictly unique and consecutive 1 to N
    ranks = gated["hiffi_xp_published_rank"].sort_values().tolist()
    expected_ranks = list(range(1, len(gated) + 1))
    assert ranks == expected_ranks, "Published ranks contain gaps or duplicates."

def test_publication_gate_blocks_unverified_identity():
    """Confirms that UNCERTAIN_NEEDS_REVIEW status does not count toward evidence tiers."""
    engine = HIFFIRankingEngine()
    test_df = pd.DataFrame([
        {
            "name": "Artist Pending Verification",
            "production_hiffi_score": 95.0,
            "spotify_monthly_listeners": 40_000_000,
            "instagram_followers": 20_000_000,
            "instagram_verification_status": "UNCERTAIN_NEEDS_REVIEW",
            "hiffi_xp_score": 96.0,
            "hiffi_xp_rank": 1
        },
        {
            "name": "Artist Fully Verified",
            "production_hiffi_score": 90.0,
            "spotify_monthly_listeners": 35_000_000,
            "instagram_followers": 15_000_000,
            "instagram_verification_status": "AUTOMATED_VERIFIED",
            "hiffi_xp_score": 91.0,
            "hiffi_xp_rank": 2
        }
    ])

    gated = engine.apply_publication_eligibility_gates(test_df)
    pending_row = gated[gated["name"] == "Artist Pending Verification"].iloc[0]
    verified_row = gated[gated["name"] == "Artist Fully Verified"].iloc[0]

    assert pending_row["platform_evidence_tier"] == 2
    assert verified_row["platform_evidence_tier"] == 3


def test_publication_gate_missing_column_fails_closed():
    """Confirms that omitting the verification column fails closed (gives 0 IG credit)."""
    engine = HIFFIRankingEngine()
    test_df = pd.DataFrame([
        {
            "name": "Artist No Status Column",
            "production_hiffi_score": 95.0,
            "spotify_monthly_listeners": 40_000_000,
            "instagram_followers": 20_000_000,
            "hiffi_xp_score": 96.0,
            "hiffi_xp_rank": 1
        }
    ])

    gated = engine.apply_publication_eligibility_gates(test_df)
    assert gated.loc[0, "platform_evidence_tier"] == 2


def test_publication_gate_zero_multiplatform_cohort():
    """Confirms an entirely Tier 1 cohort produces consecutive ranks and flags an audited shortfall."""
    engine = HIFFIRankingEngine()

    records = [
        {
            "name": f"Solo Artist {i+1}",
            "production_hiffi_score": 85.0 - i,
            "spotify_monthly_listeners": np.nan,
            "instagram_followers": np.nan,
            "instagram_verification_status": "UNVERIFIED",
            "hiffi_xp_score": 80.0 - i,
            "hiffi_xp_rank": i + 1
        }
        for i in range(25)
    ]

    tier1_df = pd.DataFrame(records)
    gated, audit = engine.apply_publication_eligibility_gates(tier1_df, return_metadata=True)

    # Must retain unique contiguous ordering 1 to 25
    assert gated["hiffi_xp_published_rank"].tolist() == list(range(1, 26))
    # All records remain Tier 1
    assert (gated["platform_evidence_tier"] == 1).all()
    # Shortfall must be explicitly reported
    assert audit["top_10_shortfall"] is True
    assert audit["top_50_shortfall"] is True