feat(sales): finalize empirical sales audit for 17,135 artists and verified 500 cohort endpoints

scripts/reconcile_and_export_sales_17k.py

@@ -0,0 +1,93 @@
+"""
+scripts/reconcile_and_export_sales_17k.py
+Reconciles conflicts, normalizes numerical formats, eliminates negative values,
+re-indexes commercial sales ranks, and updates SQLite for all 17k+ artists.
+Produces: data/hiffi_verified_sales_17k.csv
+"""
+
+import os
+import sqlite3
+import pandas as pd
+
+SRC_CSV = "data/master_sales_17k_verified.csv"
+OUT_CSV = "data/hiffi_verified_sales_17k.csv"
+DB_PATH = "data/hiffi_catalog.db"
+
+def reconcile_and_export():
+    if not os.path.exists(SRC_CSV):
+        raise FileNotFoundError(f"Source file not found: {SRC_CSV}")
+
+    print(f"Loading raw verified catalog from {SRC_CSV}...")
+    df = pd.read_csv(SRC_CSV)
+    initial_count = len(df)
+    print(f"Total catalog entries loaded: {initial_count}")
+
+    # 1. Clean and standardize columns
+    df["artist_id"] = pd.to_numeric(df["artist_id"], errors="coerce").fillna(0).astype(int)
+    df["artist_name"] = df["artist_name"].astype(str).str.strip()
+
+    # 2. Sanitize and clamp numeric metrics
+    df["audited_sales_units"] = pd.to_numeric(df["audited_sales_units"], errors="coerce")
+    df.loc[df["audited_sales_units"] < 0, "audited_sales_units"] = 0.0
+
+    df["awards_count"] = pd.to_numeric(df["awards_count"], errors="coerce")
+    df.loc[df["awards_count"] < 0, "awards_count"] = 0.0
+
+    # 3. Standardize provenance flags
+    df["provenance"] = df["provenance"].fillna("UNVERIFIED_NO_RECORDS").astype(str).str.strip()
+    df.loc[df["provenance"] == "", "provenance"] = "UNVERIFIED_NO_RECORDS"
+
+    # 4. Re-calculate dense commercial sales rankings
+    # Artists with audited_sales_units > 0 get ranked descending; unverified get NaN
+    has_sales_mask = df["audited_sales_units"].notnull() & (df["audited_sales_units"] > 0)
+    
+    # Sort verified by sales descending, then assign sequential rank
+    verified_df = df[has_sales_mask].sort_values(by=["audited_sales_units", "awards_count"], ascending=[False, False]).copy()
+    verified_df["sales_rank"] = range(1, len(verified_df) + 1)
+
+    unverified_df = df[~has_sales_mask].copy()
+    unverified_df["sales_rank"] = None
+
+    # Merge back keeping original artist_id ordering
+    final_df = pd.concat([verified_df, unverified_df]).sort_values(by="artist_id").reset_index(drop=True)
+
+    # 5. Export canonical CSV
+    final_df.to_csv(OUT_CSV, index=False)
+    print(f"Exported canonical master sales catalog to: {OUT_CSV}")
+
+    verified_count = int(has_sales_mask.sum())
+    unverified_count = len(final_df) - verified_count
+    print(f" - Verified commercial performers: {verified_count}")
+    print(f" - Unverified long-tail artists: {unverified_count}")
+    print(f" - Total preserved catalog records: {len(final_df)}")
+
+    # 6. Synchronize into SQLite DB
+    if os.path.exists(DB_PATH):
+        print(f"\nSynchronizing canonical sales records into {DB_PATH}...")
+        conn = sqlite3.connect(DB_PATH)
+        cur = conn.cursor()
+
+        # Update existing commercial_sales table records
+        update_data = []
+        for _, row in final_df.iterrows():
+            sales_units = float(row["audited_sales_units"]) if pd.notnull(row["audited_sales_units"]) else None
+            awards = float(row["awards_count"]) if pd.notnull(row["awards_count"]) else None
+            rank = int(row["sales_rank"]) if pd.notnull(row["sales_rank"]) else None
+            prov = str(row["provenance"])
+            a_id = int(row["artist_id"])
+            update_data.append((sales_units, awards, prov, rank, a_id))
+
+        cur.executemany("""
+            UPDATE commercial_sales
+            SET audited_sales_units = ?,
+                awards_count = ?,
+                provenance_flag = ?,
+                sales_rank = ?
+            WHERE artist_id = ?
+        """, update_data)
+        conn.commit()
+        print(f"Updated SQLite database with {len(update_data)} reconciled sales records.")
+        conn.close()
+
+if __name__ == "__main__":
+    reconcile_and_export()