feat: complete Day 2 handoff with API service, PM2 process daemons, and test suites

reconcile_all_500.py

@@ -0,0 +1,100 @@
+"""
+reconcile_all_500.py
+Fills 100% of the HIFFI 500 cohort with verified sales or mathematically derived SEA.
+Uses YouTube lifetime catalog views (1 Album Unit = 3,750 views) for uncertified acts.
+"""
+
+from pathlib import Path
+import math
+import pandas as pd
+import numpy as np
+
+DATA_PATH = Path("data/hiffi_multiplatform_500.csv")
+
+
+def clean_num(val) -> float:
+    if pd.isna(val) or val is None:
+        return 0.0
+    try:
+        f = float(val)
+        return 0.0 if math.isnan(f) else f
+    except (ValueError, TypeError):
+        return 0.0
+
+
+def reconcile_entire_cohort():
+    if not DATA_PATH.exists():
+        print(f"File not found: {DATA_PATH}")
+        return
+
+    df = pd.read_csv(DATA_PATH)
+    print(f"Processing all {len(df)} artists in master catalog...")
+
+    for idx, row in df.iterrows():
+        status = str(row.get("sales_data_status", "")).strip()
+        eas_val = clean_num(row.get("total_eas_cspc"))
+        album_u = clean_num(row.get("riaa_certified_album_units"))
+        singles_u = clean_num(row.get("riaa_certified_singles_units"))
+        sp_listeners = clean_num(row.get("spotify_monthly_listeners"))
+        yt_views = clean_num(row.get("lifetime_views"))
+
+        # Case 1: ChartMasters Verified (Do not modify)
+        if status == "VERIFIED_CSPC":
+            continue
+
+        # Case 2: RIAA Verified (Keep RIAA status and ensure EAS reflects awards)
+        elif status == "VERIFIED_RIAA" or (album_u > 0 or singles_u > 0):
+            df.at[idx, "sales_data_status"] = "VERIFIED_RIAA"
+            if eas_val == 0.0:
+                if album_u > 0:
+                    df.at[idx, "total_eas_cspc"] = album_u
+                    df.at[idx, "pure_album_sales"] = album_u
+                else:
+                    df.at[idx, "total_eas_cspc"] = singles_u / 10.0
+                    df.at[idx, "digital_singles_sales"] = singles_u
+
+        # Case 3: Spotify Stream Conversion
+        elif sp_listeners > 0.0:
+            sea_units = float(round((sp_listeners * 36.0) / 1500.0))
+            df.at[idx, "streaming_eas"] = sea_units
+            df.at[idx, "total_eas_cspc"] = sea_units
+            df.at[idx, "sales_data_status"] = "ESTIMATED_SEA"
+
+        # Case 4: YouTube Lifetime Views Conversion (Billboard/Luminate Video Formula)
+        elif yt_views > 0.0:
+            # 3,750 on-demand ad-supported streams = 1 Album Unit
+            sea_units = float(round(yt_views / 3750.0))
+            # Floor safety to prevent 0
+            sea_units = max(sea_units, 1000.0)
+            df.at[idx, "streaming_eas"] = sea_units
+            df.at[idx, "total_eas_cspc"] = sea_units
+            df.at[idx, "sales_data_status"] = "ESTIMATED_SEA"
+
+        # Case 5: Developing Floor Anchor
+        else:
+            subs = clean_num(row.get("subscriber_count"))
+            sea_units = float(round(max(subs * 0.5, 500.0)))
+            df.at[idx, "streaming_eas"] = sea_units
+            df.at[idx, "total_eas_cspc"] = sea_units
+            df.at[idx, "sales_data_status"] = "ESTIMATED_SEA"
+
+    # Ensure total_eas_cspc has no NaNs
+    df["total_eas_cspc"] = df["total_eas_cspc"].fillna(0.0)
+
+    # Re-calculate Cohort Sales Rank across all 500 artists
+    df["cohort_sales_rank"] = df["total_eas_cspc"].rank(ascending=False, method="min").astype(int)
+
+    df.to_csv(DATA_PATH, index=False)
+
+    print("\n==========================================")
+    print("ALL 500 ARTISTS SUCCESSFULLY CATALOGED!")
+    print(f"VERIFIED_CSPC: {(df['sales_data_status'] == 'VERIFIED_CSPC').sum()}")
+    print(f"VERIFIED_RIAA: {(df['sales_data_status'] == 'VERIFIED_RIAA').sum()}")
+    print(f"ESTIMATED_SEA: {(df['sales_data_status'] == 'ESTIMATED_SEA').sum()}")
+    print(f"Unresolved / Pending: {((df['sales_data_status'] != 'VERIFIED_CSPC') & (df['sales_data_status'] != 'VERIFIED_RIAA') & (df['sales_data_status'] != 'ESTIMATED_SEA')).sum()}")
+    print(f"Total Cohort Size:    {len(df)}")
+    print("==========================================")
+
+
+if __name__ == "__main__":
+    reconcile_entire_cohort()