hiffi-artist-intelligence/scripts/reconcile_and_export_sales_17k.py

"""
scripts/reconcile_and_export_sales_17k.py
Reconciles conflicts, normalizes numerical formats, eliminates negative values,
re-indexes commercial sales ranks, and updates SQLite for all 17k+ artists.
Produces: data/hiffi_verified_sales_17k.csv
"""

import os
import sqlite3
import pandas as pd

SRC_CSV = "data/master_sales_17k_verified.csv"
OUT_CSV = "data/hiffi_verified_sales_17k.csv"
DB_PATH = "data/hiffi_catalog.db"

def reconcile_and_export():
    if not os.path.exists(SRC_CSV):
        raise FileNotFoundError(f"Source file not found: {SRC_CSV}")

    print(f"Loading raw verified catalog from {SRC_CSV}...")
    df = pd.read_csv(SRC_CSV)
    initial_count = len(df)
    print(f"Total catalog entries loaded: {initial_count}")

    # 1. Clean and standardize columns
    df["artist_id"] = pd.to_numeric(df["artist_id"], errors="coerce").fillna(0).astype(int)
    df["artist_name"] = df["artist_name"].astype(str).str.strip()

    # 2. Sanitize and clamp numeric metrics
    df["audited_sales_units"] = pd.to_numeric(df["audited_sales_units"], errors="coerce")
    df.loc[df["audited_sales_units"] < 0, "audited_sales_units"] = 0.0

    df["awards_count"] = pd.to_numeric(df["awards_count"], errors="coerce")
    df.loc[df["awards_count"] < 0, "awards_count"] = 0.0

    # 3. Standardize provenance flags
    df["provenance"] = df["provenance"].fillna("UNVERIFIED_NO_RECORDS").astype(str).str.strip()
    df.loc[df["provenance"] == "", "provenance"] = "UNVERIFIED_NO_RECORDS"

    # 4. Re-calculate dense commercial sales rankings
    # Artists with audited_sales_units > 0 get ranked descending; unverified get NaN
    has_sales_mask = df["audited_sales_units"].notnull() & (df["audited_sales_units"] > 0)
    
    # Sort verified by sales descending, then assign sequential rank
    verified_df = df[has_sales_mask].sort_values(by=["audited_sales_units", "awards_count"], ascending=[False, False]).copy()
    verified_df["sales_rank"] = range(1, len(verified_df) + 1)

    unverified_df = df[~has_sales_mask].copy()
    unverified_df["sales_rank"] = None

    # Merge back keeping original artist_id ordering
    final_df = pd.concat([verified_df, unverified_df]).sort_values(by="artist_id").reset_index(drop=True)

    # 5. Export canonical CSV
    final_df.to_csv(OUT_CSV, index=False)
    print(f"Exported canonical master sales catalog to: {OUT_CSV}")

    verified_count = int(has_sales_mask.sum())
    unverified_count = len(final_df) - verified_count
    print(f" - Verified commercial performers: {verified_count}")
    print(f" - Unverified long-tail artists: {unverified_count}")
    print(f" - Total preserved catalog records: {len(final_df)}")

    # 6. Synchronize into SQLite DB
    if os.path.exists(DB_PATH):
        print(f"\nSynchronizing canonical sales records into {DB_PATH}...")
        conn = sqlite3.connect(DB_PATH)
        cur = conn.cursor()

        # Update existing commercial_sales table records
        update_data = []
        for _, row in final_df.iterrows():
            sales_units = float(row["audited_sales_units"]) if pd.notnull(row["audited_sales_units"]) else None
            awards = float(row["awards_count"]) if pd.notnull(row["awards_count"]) else None
            rank = int(row["sales_rank"]) if pd.notnull(row["sales_rank"]) else None
            prov = str(row["provenance"])
            a_id = int(row["artist_id"])
            update_data.append((sales_units, awards, prov, rank, a_id))

        cur.executemany("""
            UPDATE commercial_sales
            SET audited_sales_units = ?,
                awards_count = ?,
                provenance_flag = ?,
                sales_rank = ?
            WHERE artist_id = ?
        """, update_data)
        conn.commit()
        print(f"Updated SQLite database with {len(update_data)} reconciled sales records.")
        conn.close()

if __name__ == "__main__":
    reconcile_and_export()