fix: resolve governance gaps, patch credential exposure, and connect multi-platform dashboard

test_multiplatform_ranking.py

@@ -19,6 +19,7 @@ if str(PROJECT_ROOT) not in sys.path:
 from core.ranking_engine import HIFFIRankingEngine
 
 DATASET_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"
+DATASET_ANCHOR_TIME = pd.Timestamp("2026-10-26T12:00:00Z")
 
 
 @pytest.fixture(scope="module")
@@ -170,15 +171,22 @@ def test_instagram_verifier_suspicious_follower_count():
     assert followers is None or status != "AUTOMATED_VERIFIED"
 
 
-def test_spotify_fallback_null_safety():
+def test_spotify_fallback_null_safety(monkeypatch):
     """Verify Spotify embed fallback safely returns structured null dictionary on unindexed items."""
     from collectors.spotify_fallback_collector import SpotifyFallbackCollector
 
+    # Mock unindexed response to avoid brittle offline network failures
+    class MockResponse:
+        status_code = 404
+        text = "Not Found"
+
+    monkeypatch.setattr("requests.Session.get", lambda *args, **kwargs: MockResponse())
+
     fallback = SpotifyFallbackCollector()
     res = fallback.fetch_artist_fallback("Unknown Artist", "spotify:artist:invalid_or_missing_id_999999")
     assert res["spotify_monthly_listeners"] is None
     assert res["spotify_daily_change"] is None
-    assert "UNAVAILABLE" in str(res["spotify_status"]) or "EMBED" in str(res["spotify_status"])
+    assert "NOT_FOUND" in str(res["spotify_status"]) or "UNAVAILABLE" in str(res["spotify_status"])
 
 
 def test_spotify_fallback_mocked_network_error(monkeypatch):
@@ -294,9 +302,7 @@ def test_score_dominance(dataset):
 
 
 def test_kendall_tau_diagnostic(dataset):
-    """
-    Evaluates rank correlation between baseline and composite rank as a diagnostic.
-    """
+    """Evaluates rank correlation between baseline and composite rank as a diagnostic."""
     from scipy.stats import kendalltau
 
     valid = dataset.dropna(subset=["production_hiffi_rank", "hiffi_xp_rank"]).copy()
@@ -316,10 +322,7 @@ def test_score_quantile_spread(dataset):
 
 
 def test_raw_dataset_strict_nan_retention(dataset):
-    """
-    Verifies that unindexed secondary fields preserve genuine nulls
-    and guards against synthetic zeroes or string-serialized nulls ('nan', 'none', 'null').
-    """
+    """Verifies that unindexed secondary fields preserve genuine nulls and guard against synthetic zeroes."""
     secondary_metric_cols = [
         "spotify_monthly_listeners",
         "spotify_daily_change",
@@ -329,16 +332,13 @@ def test_raw_dataset_strict_nan_retention(dataset):
     for col in secondary_metric_cols:
         if col in dataset.columns:
             series = dataset[col]
-            # 1. Ensure genuine NaN values exist for unindexed artists
             assert series.isna().any(), f"Column '{col}' contains zero null values across cohort."
 
-            # 2. Guard against string serialization traps ('nan', 'None', 'null')
             string_nulls = series.dropna().astype(str).str.strip().str.lower().isin(
                 ["nan", "none", "null", "undefined", "n/a"]
             )
             assert not string_nulls.any(), f"Column '{col}' contains string-serialized null tokens."
 
-            # 3. Guard against numerical zero substitution
             numeric_vals = pd.to_numeric(series.dropna(), errors="coerce")
             if col in ["spotify_monthly_listeners", "instagram_followers"]:
                 zero_records = (numeric_vals == 0).sum()
@@ -349,9 +349,9 @@ def test_raw_dataset_strict_nan_retention(dataset):
 
 
 def test_publication_eligibility_gates(dataset):
-    """Verifies that publication gates enforce evidence tiers without altering raw scores."""
+    """Verifies that publication gates enforce evidence tiers using anchored timestamps without mutating raw scores."""
     engine = HIFFIRankingEngine()
-    gated = engine.apply_publication_eligibility_gates(dataset)
+    gated = engine.apply_publication_eligibility_gates(dataset, reference_time=DATASET_ANCHOR_TIME)
 
     # 1. Raw scores must remain unchanged
     assert (gated["hiffi_xp_score"] == dataset["hiffi_xp_score"]).all(), (
@@ -380,14 +380,44 @@ def test_publication_eligibility_gates(dataset):
     assert ranks == expected_ranks, "Published ranks contain gaps or duplicates."
 
 
+def test_publication_gate_blocks_failed_youtube_lookup():
+    """Confirms artists with status_flag == 'FAILED_LOOKUP' are disqualified from YouTube evidence credit."""
+    engine = HIFFIRankingEngine()
+    now_iso = DATASET_ANCHOR_TIME.isoformat()
+
+    test_df = pd.DataFrame([
+        {
+            "name": "Failed Lookup Artist",
+            "status_flag": "FAILED_LOOKUP",
+            "production_hiffi_score": 7.98,
+            "youtube_updated_at": now_iso,
+            "spotify_monthly_listeners": 25_000_000,
+            "spotify_updated_at": now_iso,
+            "instagram_followers": 10_000_000,
+            "instagram_verification_status": "AUTOMATED_VERIFIED",
+            "instagram_updated_at": now_iso,
+            "hiffi_xp_score": 85.0,
+            "hiffi_xp_rank": 1
+        }
+    ])
+
+    gated = engine.apply_publication_eligibility_gates(test_df, reference_time=DATASET_ANCHOR_TIME)
+    row = gated.iloc[0]
+
+    # Blocked from YouTube -> only Spotify + IG valid -> Tier 2
+    assert row["platform_evidence_tier"] == 2
+    assert "FAILED_YOUTUBE_LOOKUP" in row["evidence_exclusions"]
+
+
 def test_publication_gate_blocks_unverified_identity():
     """Confirms that UNCERTAIN_NEEDS_REVIEW status does not count toward evidence tiers."""
     engine = HIFFIRankingEngine()
-    now_iso = pd.Timestamp.now(tz="UTC").isoformat()
+    now_iso = DATASET_ANCHOR_TIME.isoformat()
 
     test_df = pd.DataFrame([
         {
             "name": "Artist Pending Verification",
+            "status_flag": "COMPLETE",
             "production_hiffi_score": 95.0,
             "youtube_updated_at": now_iso,
             "spotify_monthly_listeners": 40_000_000,
@@ -400,6 +430,7 @@ def test_publication_gate_blocks_unverified_identity():
         },
         {
             "name": "Artist Fully Verified",
+            "status_flag": "COMPLETE",
             "production_hiffi_score": 90.0,
             "youtube_updated_at": now_iso,
             "spotify_monthly_listeners": 35_000_000,
@@ -412,7 +443,7 @@ def test_publication_gate_blocks_unverified_identity():
         }
     ])
 
-    gated = engine.apply_publication_eligibility_gates(test_df)
+    gated = engine.apply_publication_eligibility_gates(test_df, reference_time=DATASET_ANCHOR_TIME)
     pending_row = gated[gated["name"] == "Artist Pending Verification"].iloc[0]
     verified_row = gated[gated["name"] == "Artist Fully Verified"].iloc[0]
 
@@ -423,11 +454,12 @@ def test_publication_gate_blocks_unverified_identity():
 def test_publication_gate_missing_column_fails_closed():
     """Confirms that omitting the verification column fails closed (gives 0 IG credit)."""
     engine = HIFFIRankingEngine()
-    now_iso = pd.Timestamp.now(tz="UTC").isoformat()
+    now_iso = DATASET_ANCHOR_TIME.isoformat()
 
     test_df = pd.DataFrame([
         {
             "name": "Artist No Status Column",
+            "status_flag": "COMPLETE",
             "production_hiffi_score": 95.0,
             "youtube_updated_at": now_iso,
             "spotify_monthly_listeners": 40_000_000,
@@ -439,18 +471,19 @@ def test_publication_gate_missing_column_fails_closed():
         }
     ])
 
-    gated = engine.apply_publication_eligibility_gates(test_df)
+    gated = engine.apply_publication_eligibility_gates(test_df, reference_time=DATASET_ANCHOR_TIME)
     assert gated.loc[0, "platform_evidence_tier"] == 2
 
 
 def test_publication_gate_zero_multiplatform_cohort():
     """Confirms an entirely Tier 1 cohort produces consecutive ranks and flags an audited shortfall."""
     engine = HIFFIRankingEngine()
-    now_iso = pd.Timestamp.now(tz="UTC").isoformat()
+    now_iso = DATASET_ANCHOR_TIME.isoformat()
 
     records = [
         {
             "name": f"Solo Artist {i+1}",
+            "status_flag": "COMPLETE",
             "production_hiffi_score": 85.0 - i,
             "youtube_updated_at": now_iso,
             "spotify_monthly_listeners": np.nan,
@@ -465,13 +498,10 @@ def test_publication_gate_zero_multiplatform_cohort():
     ]
 
     tier1_df = pd.DataFrame(records)
-    gated, audit = engine.apply_publication_eligibility_gates(tier1_df, return_metadata=True)
+    gated, audit = engine.apply_publication_eligibility_gates(tier1_df, reference_time=DATASET_ANCHOR_TIME, return_metadata=True)
 
-    # Must retain unique contiguous ordering 1 to 25
     assert gated["hiffi_xp_published_rank"].tolist() == list(range(1, 26))
-    # All records remain Tier 1
     assert (gated["platform_evidence_tier"] == 1).all()
-    # Shortfall must be explicitly reported
     assert audit["top_10_shortfall"] is True
     assert audit["top_50_shortfall"] is True
 
@@ -479,11 +509,11 @@ def test_publication_gate_zero_multiplatform_cohort():
 def test_freshness_gate_stale_timestamps_demote_tier():
     """Validates that metrics older than threshold lose evidence tier credit while retaining raw score."""
     engine = HIFFIRankingEngine()
-    ref_time = pd.Timestamp("2026-10-26T12:00:00Z")
+    ref_time = DATASET_ANCHOR_TIME
 
-    # Artist has all metrics present, but Spotify is 10 days old (>7d) and IG is 20 days old (>14d)
     test_df = pd.DataFrame([{
         "name": "Artist Stale Telemetry",
+        "status_flag": "COMPLETE",
         "production_hiffi_score": 90.0,
         "youtube_updated_at": "2026-10-25T12:00:00Z",  # 1 day old (Fresh)
         "spotify_monthly_listeners": 25_000_000,
@@ -498,7 +528,6 @@ def test_freshness_gate_stale_timestamps_demote_tier():
     gated = engine.apply_publication_eligibility_gates(test_df, reference_time=ref_time)
     row = gated.iloc[0]
 
-    # Only YouTube is fresh; Tier must demote to 1
     assert row["platform_evidence_tier"] == 1
     assert row["hiffi_xp_score"] == 88.5
     assert "STALE_OR_MISSING_SPOTIFY_TIMESTAMP" in row["evidence_exclusions"]
@@ -508,10 +537,11 @@ def test_freshness_gate_stale_timestamps_demote_tier():
 def test_freshness_gate_missing_or_malformed_timestamp():
     """Fails closed when timestamp string is empty, null, or malformed."""
     engine = HIFFIRankingEngine()
-    ref_time = pd.Timestamp("2026-10-26T12:00:00Z")
+    ref_time = DATASET_ANCHOR_TIME
 
     test_df = pd.DataFrame([{
         "name": "Artist Malformed Timestamp",
+        "status_flag": "COMPLETE",
         "production_hiffi_score": 85.0,
         "youtube_updated_at": "invalid_date_format",
         "spotify_monthly_listeners": 15_000_000,
@@ -526,7 +556,6 @@ def test_freshness_gate_missing_or_malformed_timestamp():
     gated = engine.apply_publication_eligibility_gates(test_df, reference_time=ref_time)
     row = gated.iloc[0]
 
-    # Only Instagram is valid & fresh -> Tier 1
     assert row["platform_evidence_tier"] == 1
     assert "STALE_OR_MISSING_YOUTUBE_TIMESTAMP" in row["evidence_exclusions"]
     assert "STALE_OR_MISSING_SPOTIFY_TIMESTAMP" in row["evidence_exclusions"]
@@ -538,6 +567,7 @@ def test_freshness_gate_missing_timestamp_columns_fail_closed():
 
     test_df = pd.DataFrame([{
         "name": "Artist No Timestamps",
+        "status_flag": "COMPLETE",
         "production_hiffi_score": 90.0,
         "spotify_monthly_listeners": 20_000_000,
         "instagram_followers": 10_000_000,
@@ -546,7 +576,7 @@ def test_freshness_gate_missing_timestamp_columns_fail_closed():
         "hiffi_xp_rank": 1
     }])
 
-    gated = engine.apply_publication_eligibility_gates(test_df)
+    gated = engine.apply_publication_eligibility_gates(test_df, reference_time=DATASET_ANCHOR_TIME)
     row = gated.iloc[0]
 
     assert row["platform_evidence_tier"] == 0
@@ -591,5 +621,4 @@ def test_anomaly_detection_flags_suspicious_patterns():
     assert flagged.loc[0, "anomaly_flags"] == "CLEAN"
     assert "SUSPICIOUS_IG_TO_YT_RATIO" in flagged.loc[1, "anomaly_flags"]
     assert "EXTREME_SPOTIFY_VELOCITY_RATIO" in flagged.loc[2, "anomaly_flags"]
-    # Raw scores must remain unaffected
     assert flagged.loc[1, "hiffi_xp_score"] == 70.0