feat: complete Day 2 handoff with API service, PM2 process daemons, and test suites

reconcile_sales_telemetry.py

@@ -0,0 +1,126 @@
+"""
+reconcile_sales_telemetry.py
+Hardened reconciliation engine for HIFFI multi-platform dataset.
+Applies industry-standard streaming conversions (Tier 3: ESTIMATED_SEA)
+to uncertified acts and re-indexes cohort sales rankings across all 500 artists.
+"""
+
+from pathlib import Path
+import math
+import numpy as np
+import pandas as pd
+
+DATA_PATH = Path("data/hiffi_multiplatform_500.csv")
+
+
+def clean_num(val) -> float:
+    """Safely converts any input into a float, returning 0.0 for NaNs or None."""
+    if pd.isna(val) or val is None:
+        return 0.0
+    try:
+        f = float(val)
+        return 0.0 if math.isnan(f) else f
+    except (ValueError, TypeError):
+        return 0.0
+
+
+def reconcile_dataset():
+    if not DATA_PATH.exists():
+        print(f"File not found: {DATA_PATH}")
+        return
+
+    df = pd.read_csv(DATA_PATH)
+    print(f"Loaded master dataset with {len(df)} artists.")
+
+    # Standardize column types
+    numeric_fields = [
+        "total_eas_cspc",
+        "pure_album_sales",
+        "streaming_eas",
+        "physical_singles_sales",
+        "digital_singles_sales",
+        "riaa_certified_album_units",
+        "riaa_certified_singles_units",
+        "spotify_monthly_listeners",
+        "lifetime_views",
+    ]
+    for col in numeric_fields:
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+        else:
+            df[col] = np.nan
+
+    for idx, row in df.iterrows():
+        status = str(row.get("sales_data_status", ""))
+
+        # 1. Tier 1: CSPC Verified (Untouched)
+        if status == "VERIFIED_CSPC":
+            continue
+
+        # 2. Tier 2: RIAA Verified (Standardize EAS Anchor)
+        elif status == "VERIFIED_RIAA":
+            album_u = clean_num(row.get("riaa_certified_album_units"))
+            singles_u = clean_num(row.get("riaa_certified_singles_units"))
+
+            current_eas = clean_num(row.get("total_eas_cspc"))
+            if current_eas == 0.0:
+                if album_u > 0:
+                    df.at[idx, "total_eas_cspc"] = album_u
+                    df.at[idx, "pure_album_sales"] = album_u
+                else:
+                    # 10 digital singles = 1 album equivalent
+                    tea = singles_u / 10.0
+                    df.at[idx, "total_eas_cspc"] = tea
+                    df.at[idx, "digital_singles_sales"] = singles_u
+
+        # 3. Tier 3: Apply Streaming SEA fallback for uncertified acts
+        else:
+            sp_listeners = clean_num(row.get("spotify_monthly_listeners"))
+            yt_views = clean_num(row.get("lifetime_views"))
+
+            if sp_listeners > 0.0:
+                # 1 monthly listener generates ~36 annual on-demand audio streams
+                # 1,500 audio streams = 1 Album Unit (SEA)
+                estimated_annual_streams = sp_listeners * 36.0
+                sea_units = float(round(estimated_annual_streams / 1500.0))
+
+                df.at[idx, "streaming_eas"] = sea_units
+                df.at[idx, "total_eas_cspc"] = sea_units
+                df.at[idx, "pure_album_sales"] = 0.0
+                df.at[idx, "physical_singles_sales"] = 0.0
+                df.at[idx, "digital_singles_sales"] = 0.0
+                df.at[idx, "sales_data_status"] = "ESTIMATED_SEA"
+            elif yt_views > 0.0:
+                # Video conversion: 3,750 ad-supported video streams = 1 Album Unit
+                sea_units = float(round(yt_views / 3750.0))
+                df.at[idx, "streaming_eas"] = sea_units
+                df.at[idx, "total_eas_cspc"] = sea_units
+                df.at[idx, "pure_album_sales"] = 0.0
+                df.at[idx, "physical_singles_sales"] = 0.0
+                df.at[idx, "digital_singles_sales"] = 0.0
+                df.at[idx, "sales_data_status"] = "ESTIMATED_SEA"
+            else:
+                # Tier 4: No certified sales, no streams
+                df.at[idx, "sales_data_status"] = "UNRANKED_EMERGING"
+                df.at[idx, "total_eas_cspc"] = 0.0
+
+    # Ensure total_eas_cspc has no NaNs
+    df["total_eas_cspc"] = df["total_eas_cspc"].fillna(0.0)
+
+    # Re-calculate Cohort Sales Rank across all 500 artists
+    df["cohort_sales_rank"] = df["total_eas_cspc"].rank(ascending=False, method="min")
+
+    df.to_csv(DATA_PATH, index=False)
+
+    print("\n==========================================")
+    print("Reconciliation Complete!")
+    print(f"VERIFIED_CSPC:     {(df['sales_data_status'] == 'VERIFIED_CSPC').sum()}")
+    print(f"VERIFIED_RIAA:     {(df['sales_data_status'] == 'VERIFIED_RIAA').sum()}")
+    print(f"ESTIMATED_SEA:     {(df['sales_data_status'] == 'ESTIMATED_SEA').sum()}")
+    print(f"UNRANKED_EMERGING: {(df['sales_data_status'] == 'UNRANKED_EMERGING').sum()}")
+    print(f"Total Cohort Size: {len(df)}")
+    print("==========================================")
+
+
+if __name__ == "__main__":
+    reconcile_dataset()