hiffi-artist-intelligence/scripts/monitor_ranking_drift.py

"""
scripts/monitor_ranking_drift.py
T15 - Separate Invariant Checks for:
  1. The 500-Artist Live Telemetry Cohort
  2. The 17,135-Artist Commercial Pure Sales Catalog
"""

import os
import sqlite3
from datetime import datetime

DB_PATH = "data/hiffi_catalog.db"
LOG_PATH = "cron_ranking.log"

def run_isolated_drift_checks():
    if not os.path.exists(DB_PATH):
        raise FileNotFoundError(f"Database not found at {DB_PATH}")

    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    print("=" * 65)
    print("HIFFI INTELLIGENCE: DUAL-SYSTEM INVARIANT AUDIT")
    print("=" * 65)

    cohort_issues = []
    sales_issues = []

    # ==============================================================
    # SYSTEM 1: 500-ARTIST LIVE TELEMETRY COHORT
    # ==============================================================
    cur.execute("SELECT COUNT(*) FROM artists WHERE image_verified_flag = 1;")
    verified_cohort_count = cur.fetchone()[0]

    if verified_cohort_count != 500:
        cohort_issues.append(f"Cohort size mismatch: {verified_cohort_count} (Expected strictly 500)")

    # Check telemetry coverage for the 500
    cur.execute("""
        SELECT 
            COUNT(t.artist_id),
            SUM(CASE WHEN t.growth_velocity_7d IS NOT NULL THEN 1 ELSE 0 END),
            SUM(CASE WHEN t.growth_velocity_30d IS NOT NULL THEN 1 ELSE 0 END)
        FROM telemetry_metrics t
        JOIN artists a ON t.artist_id = a.artist_id
        WHERE a.image_verified_flag = 1;
    """)
    telemetry_count, has_7d, has_30d = cur.fetchone()

    if telemetry_count != 500 or has_7d != 500 or has_30d != 500:
        cohort_issues.append(f"Telemetry gap: {telemetry_count}/500 present, {has_7d} with 7d, {has_30d} with 30d")

    # Check 500 snapshots
    cur.execute("""
        SELECT COUNT(DISTINCT s.artist_id), COUNT(*) 
        FROM artist_metrics_snapshots s
        JOIN artists a ON s.artist_id = a.artist_id
        WHERE a.image_verified_flag = 1;
    """)
    snapshot_artists, total_snapshots = cur.fetchone()

    # Check tier breakdown within the 500
    cur.execute("""
        SELECT catalog_intelligence_tier, COUNT(*)
        FROM rankings r
        JOIN artists a ON r.artist_id = a.artist_id
        WHERE a.image_verified_flag = 1
        GROUP BY catalog_intelligence_tier;
    """)
    cohort_tiers = dict(cur.fetchall())

    # ==============================================================
    # SYSTEM 2: 17,135 MASTER COMMERCIAL SALES CATALOG
    # ==============================================================
    cur.execute("SELECT COUNT(*) FROM commercial_sales;")
    total_sales_records = cur.fetchone()[0]

    cur.execute("""
        SELECT 
            MIN(sales_rank),
            MAX(sales_rank),
            COUNT(DISTINCT sales_rank),
            SUM(CASE WHEN streaming_eas != 0.0 THEN 1 ELSE 0 END),
            ROUND(SUM(audited_sales_units), 2)
        FROM commercial_sales;
    """)
    min_rank, max_rank, distinct_ranks, non_zero_streaming, total_sales_volume = cur.fetchone()

    if total_sales_records != 17135:
        sales_issues.append(f"Sales catalog size mismatch: {total_sales_records} (Expected 17,135)")

    if min_rank != 1 or max_rank != 17135 or distinct_ranks != 17135:
        sales_issues.append(f"Sales rank gap: min={min_rank}, max={max_rank}, distinct={distinct_ranks}")

    if non_zero_streaming > 0:
        sales_issues.append(f"Synthetic streaming detected: {non_zero_streaming} records violate pure sales rule")

    conn.close()

    # Output results
    cohort_status = "HEALTHY" if not cohort_issues else "ALERT"
    sales_status = "HEALTHY" if not sales_issues else "ALERT"

    print(f"\n[1. Cohort Telemetry System - 500 Artists]")
    print(f"  Status        : {cohort_status}")
    print(f"  Cohort Size   : {verified_cohort_count} verified artists")
    print(f"  Tiers         : {cohort_tiers}")
    print(f"  7d/30d Deltas : 100% complete ({has_7d}/500)")
    print(f"  Snapshots     : {total_snapshots:,} rows across {snapshot_artists} artists")
    if cohort_issues:
        for err in cohort_issues:
            print(f"  [!] {err}")

    print(f"\n[2. Commercial Pure Sales System - 17,135 Artists]")
    print(f"  Status        : {sales_status}")
    print(f"  Catalog Scope : {total_sales_records:,} artists")
    print(f"  Sales Ranks   : #{min_rank} to #{max_rank:,} (Continuous, 0 duplicates)")
    print(f"  Pure Volume   : {total_sales_volume:,.2f} units (Streaming SEA/TEA = 0.0)")
    if sales_issues:
        for err in sales_issues:
            print(f"  [!] {err}")

    # Write separate log entries
    log_line = (
        f"[{timestamp}] T15_MONITORING | "
        f"COHORT_500: status={cohort_status}, errors={len(cohort_issues)} | "
        f"SALES_17K: status={sales_status}, errors={len(sales_issues)}\n"
    )
    with open(LOG_PATH, "a", encoding="utf-8") as f:
        f.write(log_line)

    print("\n" + "=" * 65)
    print("Dual audit complete. Both systems isolated and verified.")
    print("=" * 65)

if __name__ == "__main__":
    run_isolated_drift_checks()