hiffi-artist-intelligence/reconcile_sales_telemetry.py

"""
reconcile_sales_telemetry.py
Hardened reconciliation engine for HIFFI multi-platform dataset.
Applies industry-standard streaming conversions (Tier 3: ESTIMATED_SEA)
to uncertified acts and re-indexes cohort sales rankings across all 500 artists.
"""

from pathlib import Path
import math
import numpy as np
import pandas as pd

DATA_PATH = Path("data/hiffi_multiplatform_500.csv")


def clean_num(val) -> float:
    """Safely converts any input into a float, returning 0.0 for NaNs or None."""
    if pd.isna(val) or val is None:
        return 0.0
    try:
        f = float(val)
        return 0.0 if math.isnan(f) else f
    except (ValueError, TypeError):
        return 0.0


def reconcile_dataset():
    if not DATA_PATH.exists():
        print(f"File not found: {DATA_PATH}")
        return

    df = pd.read_csv(DATA_PATH)
    print(f"Loaded master dataset with {len(df)} artists.")

    # Standardize column types
    numeric_fields = [
        "total_eas_cspc",
        "pure_album_sales",
        "streaming_eas",
        "physical_singles_sales",
        "digital_singles_sales",
        "riaa_certified_album_units",
        "riaa_certified_singles_units",
        "spotify_monthly_listeners",
        "lifetime_views",
    ]
    for col in numeric_fields:
        if col in df.columns:
            df[col] = pd.to_numeric(df[col], errors="coerce")
        else:
            df[col] = np.nan

    for idx, row in df.iterrows():
        status = str(row.get("sales_data_status", ""))

        # 1. Tier 1: CSPC Verified (Untouched)
        if status == "VERIFIED_CSPC":
            continue

        # 2. Tier 2: RIAA Verified (Standardize EAS Anchor)
        elif status == "VERIFIED_RIAA":
            album_u = clean_num(row.get("riaa_certified_album_units"))
            singles_u = clean_num(row.get("riaa_certified_singles_units"))

            current_eas = clean_num(row.get("total_eas_cspc"))
            if current_eas == 0.0:
                if album_u > 0:
                    df.at[idx, "total_eas_cspc"] = album_u
                    df.at[idx, "pure_album_sales"] = album_u
                else:
                    # 10 digital singles = 1 album equivalent
                    tea = singles_u / 10.0
                    df.at[idx, "total_eas_cspc"] = tea
                    df.at[idx, "digital_singles_sales"] = singles_u

        # 3. Tier 3: Apply Streaming SEA fallback for uncertified acts
        else:
            sp_listeners = clean_num(row.get("spotify_monthly_listeners"))
            yt_views = clean_num(row.get("lifetime_views"))

            if sp_listeners > 0.0:
                # 1 monthly listener generates ~36 annual on-demand audio streams
                # 1,500 audio streams = 1 Album Unit (SEA)
                estimated_annual_streams = sp_listeners * 36.0
                sea_units = float(round(estimated_annual_streams / 1500.0))

                df.at[idx, "streaming_eas"] = sea_units
                df.at[idx, "total_eas_cspc"] = sea_units
                df.at[idx, "pure_album_sales"] = 0.0
                df.at[idx, "physical_singles_sales"] = 0.0
                df.at[idx, "digital_singles_sales"] = 0.0
                df.at[idx, "sales_data_status"] = "ESTIMATED_SEA"
            elif yt_views > 0.0:
                # Video conversion: 3,750 ad-supported video streams = 1 Album Unit
                sea_units = float(round(yt_views / 3750.0))
                df.at[idx, "streaming_eas"] = sea_units
                df.at[idx, "total_eas_cspc"] = sea_units
                df.at[idx, "pure_album_sales"] = 0.0
                df.at[idx, "physical_singles_sales"] = 0.0
                df.at[idx, "digital_singles_sales"] = 0.0
                df.at[idx, "sales_data_status"] = "ESTIMATED_SEA"
            else:
                # Tier 4: No certified sales, no streams
                df.at[idx, "sales_data_status"] = "UNRANKED_EMERGING"
                df.at[idx, "total_eas_cspc"] = 0.0

    # Ensure total_eas_cspc has no NaNs
    df["total_eas_cspc"] = df["total_eas_cspc"].fillna(0.0)

    # Re-calculate Cohort Sales Rank across all 500 artists
    df["cohort_sales_rank"] = df["total_eas_cspc"].rank(ascending=False, method="min")

    df.to_csv(DATA_PATH, index=False)

    print("\n==========================================")
    print("Reconciliation Complete!")
    print(f"VERIFIED_CSPC:     {(df['sales_data_status'] == 'VERIFIED_CSPC').sum()}")
    print(f"VERIFIED_RIAA:     {(df['sales_data_status'] == 'VERIFIED_RIAA').sum()}")
    print(f"ESTIMATED_SEA:     {(df['sales_data_status'] == 'ESTIMATED_SEA').sum()}")
    print(f"UNRANKED_EMERGING: {(df['sales_data_status'] == 'UNRANKED_EMERGING').sum()}")
    print(f"Total Cohort Size: {len(df)}")
    print("==========================================")


if __name__ == "__main__":
    reconcile_dataset()