""" scripts/backfill_catalog_data.py Dual-source ingestion engine for master catalog and multiplatform telemetry. Enforces the Zero-Null Standard across all numeric and velocity fields. """ import sys import re from pathlib import Path import pandas as pd import numpy as np ROOT_DIR = Path(__file__).resolve().parent.parent if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) from core.database import get_connection MASTER_PATH = ROOT_DIR / "data" / "master_artists_intelligence_17k_final.csv" MULTIPLATFORM_PATH = ROOT_DIR / "data" / "hiffi_multiplatform_500.csv" def clean_str(val, default=""): if pd.isna(val) or val is None: return default s = str(val).strip() return default if s.lower() in ("nan", "none", "null") else s def clean_int(val, default=0): if pd.isna(val) or val is None: return default try: return int(float(val)) except (ValueError, TypeError): return default def clean_float(val, default=0.0): if pd.isna(val) or val is None: return default try: f = float(val) return default if np.isnan(f) or np.isinf(f) else f except (ValueError, TypeError): return default def generate_slug_id(name: str) -> str: slug = re.sub(r"[^a-zA-Z0-9]+", "_", name.strip().lower()).strip("_") return f"art_{slug}" if slug else "art_unknown" def backfill(): conn = get_connection() cursor = conn.cursor() print("Beginning master catalog ingestion (17k)...") name_to_id = {} if MASTER_PATH.exists(): df_master = pd.read_csv(MASTER_PATH, low_memory=False) for _, row in df_master.iterrows(): artist_id = clean_str(row.get("artist_id")) artist_name = clean_str(row.get("artist_name")) if not artist_id or not artist_name: continue name_to_id[artist_name.lower()] = artist_id cursor.execute( """ INSERT INTO artists (artist_id, artist_name, country_origin) VALUES (?, ?, ?) ON CONFLICT(artist_id) DO UPDATE SET artist_name = excluded.artist_name; """, (artist_id, artist_name, "Unknown"), ) cursor.execute( """ INSERT INTO rankings (artist_id, catalog_rank, tier_rank, catalog_intelligence_tier) VALUES (?, ?, ?, ?) ON CONFLICT(artist_id) DO UPDATE SET catalog_rank = excluded.catalog_rank, tier_rank = excluded.tier_rank, catalog_intelligence_tier = excluded.catalog_intelligence_tier; """, ( artist_id, clean_int(row.get("catalog_rank"), 99999), clean_int(row.get("tier_rank"), 0), clean_str(row.get("catalog_intelligence_tier"), "Emerging"), ), ) cursor.execute( """ INSERT INTO commercial_sales (artist_id, audited_sales_units, awards_count, provenance, sales_rank) VALUES (?, ?, ?, ?, ?) ON CONFLICT(artist_id) DO UPDATE SET audited_sales_units = excluded.audited_sales_units, awards_count = excluded.awards_count, provenance = excluded.provenance, sales_rank = excluded.sales_rank; """, ( artist_id, clean_float(row.get("audited_sales_units"), 0.0), clean_int(row.get("awards_count"), 0), clean_str(row.get("provenance"), "N/A"), clean_int(row.get("sales_rank"), 0), ), ) cursor.execute( """ INSERT INTO telemetry_metrics ( artist_id, subscribers_count, spotify_monthly_listeners, growth_velocity_7d, growth_velocity_30d ) VALUES (?, ?, ?, ?, ?) ON CONFLICT(artist_id) DO NOTHING; """, ( artist_id, 0, clean_int(row.get("spotify_monthly_listeners"), 0), 0.0, 0.0, ), ) conn.commit() print(f"Master catalog processed: {len(df_master)} records.") print("Processing multiplatform 500 cohort...") if MULTIPLATFORM_PATH.exists(): df_500 = pd.read_csv(MULTIPLATFORM_PATH) for _, row in df_500.iterrows(): name = clean_str(row.get("name")) if not name: continue artist_id = name_to_id.get(name.lower()) if not artist_id: raw_ident = clean_str(row.get("identifier")) artist_id = raw_ident if raw_ident else generate_slug_id(name) name_to_id[name.lower()] = artist_id cursor.execute( """ INSERT INTO artists (artist_id, artist_name) VALUES (?, ?) ON CONFLICT(artist_id) DO UPDATE SET artist_name = excluded.artist_name; """, (artist_id, name), ) hiffi_rank = clean_int( row.get("hiffi_xp_published_rank") or row.get("production_hiffi_rank") or row.get("hiffi_rank"), 99999, ) cursor.execute( """ INSERT INTO rankings (artist_id, catalog_rank, tier_rank, catalog_intelligence_tier) VALUES (?, ?, ?, ?) ON CONFLICT(artist_id) DO UPDATE SET catalog_rank = excluded.catalog_rank; """, (artist_id, hiffi_rank, 1, "Top 500"), ) sales_units = clean_float(row.get("streaming_eas"), 0.0) sales_rank = clean_int(row.get("cohort_sales_rank") or row.get("chartmasters_sales_rank"), 0) cursor.execute( """ INSERT INTO commercial_sales (artist_id, audited_sales_units, provenance, sales_rank) VALUES (?, ?, ?, ?) ON CONFLICT(artist_id) DO UPDATE SET audited_sales_units = CASE WHEN excluded.audited_sales_units > 0.0 THEN excluded.audited_sales_units ELSE commercial_sales.audited_sales_units END, sales_rank = CASE WHEN excluded.sales_rank > 0 THEN excluded.sales_rank ELSE commercial_sales.sales_rank END; """, (artist_id, sales_units, "Multiplatform Pipeline", sales_rank), ) subs = clean_int(row.get("subscriber_count"), 0) listeners = clean_int(row.get("spotify_monthly_listeners"), 0) vel_7d = clean_float(row.get("daily_view_velocity"), 0.0) vel_30d = clean_float(row.get("momentum_30d"), 0.0) cursor.execute( """ INSERT INTO telemetry_metrics ( artist_id, subscribers_count, spotify_monthly_listeners, growth_velocity_7d, growth_velocity_30d ) VALUES (?, ?, ?, ?, ?) ON CONFLICT(artist_id) DO UPDATE SET subscribers_count = excluded.subscribers_count, spotify_monthly_listeners = CASE WHEN excluded.spotify_monthly_listeners > 0 THEN excluded.spotify_monthly_listeners ELSE telemetry_metrics.spotify_monthly_listeners END, growth_velocity_7d = excluded.growth_velocity_7d, growth_velocity_30d = excluded.growth_velocity_30d; """, (artist_id, subs, listeners, vel_7d, vel_30d), ) conn.commit() print(f"Multiplatform 500 cohort ingested: {len(df_500)} records.") conn.close() print("Zero-null backfill completed successfully.") if __name__ == "__main__": backfill()