217 lines
7.9 KiB
Python
217 lines
7.9 KiB
Python
|
"""
|
||
|
|
scripts/backfill_catalog_data.py
|
||
|
|
Dual-source ingestion engine for master catalog and multiplatform telemetry.
|
||
|
|
Enforces the Zero-Null Standard across all numeric and velocity fields.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import sys
|
||
|
|
import re
|
||
|
|
from pathlib import Path
|
||
|
|
import pandas as pd
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
ROOT_DIR = Path(__file__).resolve().parent.parent
|
||
|
|
if str(ROOT_DIR) not in sys.path:
|
||
|
|
sys.path.insert(0, str(ROOT_DIR))
|
||
|
|
|
||
|
|
from core.database import get_connection
|
||
|
|
|
||
|
|
MASTER_PATH = ROOT_DIR / "data" / "master_artists_intelligence_17k_final.csv"
|
||
|
|
MULTIPLATFORM_PATH = ROOT_DIR / "data" / "hiffi_multiplatform_500.csv"
|
||
|
|
|
||
|
|
|
||
|
|
def clean_str(val, default=""):
|
||
|
|
if pd.isna(val) or val is None:
|
||
|
|
return default
|
||
|
|
s = str(val).strip()
|
||
|
|
return default if s.lower() in ("nan", "none", "null") else s
|
||
|
|
|
||
|
|
|
||
|
|
def clean_int(val, default=0):
|
||
|
|
if pd.isna(val) or val is None:
|
||
|
|
return default
|
||
|
|
try:
|
||
|
|
return int(float(val))
|
||
|
|
except (ValueError, TypeError):
|
||
|
|
return default
|
||
|
|
|
||
|
|
|
||
|
|
def clean_float(val, default=0.0):
|
||
|
|
if pd.isna(val) or val is None:
|
||
|
|
return default
|
||
|
|
try:
|
||
|
|
f = float(val)
|
||
|
|
return default if np.isnan(f) or np.isinf(f) else f
|
||
|
|
except (ValueError, TypeError):
|
||
|
|
return default
|
||
|
|
|
||
|
|
|
||
|
|
def generate_slug_id(name: str) -> str:
|
||
|
|
slug = re.sub(r"[^a-zA-Z0-9]+", "_", name.strip().lower()).strip("_")
|
||
|
|
return f"art_{slug}" if slug else "art_unknown"
|
||
|
|
|
||
|
|
|
||
|
|
def backfill():
|
||
|
|
conn = get_connection()
|
||
|
|
cursor = conn.cursor()
|
||
|
|
|
||
|
|
print("Beginning master catalog ingestion (17k)...")
|
||
|
|
name_to_id = {}
|
||
|
|
|
||
|
|
if MASTER_PATH.exists():
|
||
|
|
df_master = pd.read_csv(MASTER_PATH, low_memory=False)
|
||
|
|
for _, row in df_master.iterrows():
|
||
|
|
artist_id = clean_str(row.get("artist_id"))
|
||
|
|
artist_name = clean_str(row.get("artist_name"))
|
||
|
|
if not artist_id or not artist_name:
|
||
|
|
continue
|
||
|
|
|
||
|
|
name_to_id[artist_name.lower()] = artist_id
|
||
|
|
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO artists (artist_id, artist_name, country_origin)
|
||
|
|
VALUES (?, ?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO UPDATE SET
|
||
|
|
artist_name = excluded.artist_name;
|
||
|
|
""",
|
||
|
|
(artist_id, artist_name, "Unknown"),
|
||
|
|
)
|
||
|
|
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO rankings (artist_id, catalog_rank, tier_rank, catalog_intelligence_tier)
|
||
|
|
VALUES (?, ?, ?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO UPDATE SET
|
||
|
|
catalog_rank = excluded.catalog_rank,
|
||
|
|
tier_rank = excluded.tier_rank,
|
||
|
|
catalog_intelligence_tier = excluded.catalog_intelligence_tier;
|
||
|
|
""",
|
||
|
|
(
|
||
|
|
artist_id,
|
||
|
|
clean_int(row.get("catalog_rank"), 99999),
|
||
|
|
clean_int(row.get("tier_rank"), 0),
|
||
|
|
clean_str(row.get("catalog_intelligence_tier"), "Emerging"),
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO commercial_sales (artist_id, audited_sales_units, awards_count, provenance, sales_rank)
|
||
|
|
VALUES (?, ?, ?, ?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO UPDATE SET
|
||
|
|
audited_sales_units = excluded.audited_sales_units,
|
||
|
|
awards_count = excluded.awards_count,
|
||
|
|
provenance = excluded.provenance,
|
||
|
|
sales_rank = excluded.sales_rank;
|
||
|
|
""",
|
||
|
|
(
|
||
|
|
artist_id,
|
||
|
|
clean_float(row.get("audited_sales_units"), 0.0),
|
||
|
|
clean_int(row.get("awards_count"), 0),
|
||
|
|
clean_str(row.get("provenance"), "N/A"),
|
||
|
|
clean_int(row.get("sales_rank"), 0),
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO telemetry_metrics (
|
||
|
|
artist_id, subscribers_count, spotify_monthly_listeners,
|
||
|
|
growth_velocity_7d, growth_velocity_30d
|
||
|
|
)
|
||
|
|
VALUES (?, ?, ?, ?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO NOTHING;
|
||
|
|
""",
|
||
|
|
(
|
||
|
|
artist_id,
|
||
|
|
0,
|
||
|
|
clean_int(row.get("spotify_monthly_listeners"), 0),
|
||
|
|
0.0,
|
||
|
|
0.0,
|
||
|
|
),
|
||
|
|
)
|
||
|
|
conn.commit()
|
||
|
|
print(f"Master catalog processed: {len(df_master)} records.")
|
||
|
|
|
||
|
|
print("Processing multiplatform 500 cohort...")
|
||
|
|
if MULTIPLATFORM_PATH.exists():
|
||
|
|
df_500 = pd.read_csv(MULTIPLATFORM_PATH)
|
||
|
|
for _, row in df_500.iterrows():
|
||
|
|
name = clean_str(row.get("name"))
|
||
|
|
if not name:
|
||
|
|
continue
|
||
|
|
|
||
|
|
artist_id = name_to_id.get(name.lower())
|
||
|
|
if not artist_id:
|
||
|
|
raw_ident = clean_str(row.get("identifier"))
|
||
|
|
artist_id = raw_ident if raw_ident else generate_slug_id(name)
|
||
|
|
name_to_id[name.lower()] = artist_id
|
||
|
|
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO artists (artist_id, artist_name)
|
||
|
|
VALUES (?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO UPDATE SET
|
||
|
|
artist_name = excluded.artist_name;
|
||
|
|
""",
|
||
|
|
(artist_id, name),
|
||
|
|
)
|
||
|
|
|
||
|
|
hiffi_rank = clean_int(
|
||
|
|
row.get("hiffi_xp_published_rank") or row.get("production_hiffi_rank") or row.get("hiffi_rank"),
|
||
|
|
99999,
|
||
|
|
)
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO rankings (artist_id, catalog_rank, tier_rank, catalog_intelligence_tier)
|
||
|
|
VALUES (?, ?, ?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO UPDATE SET
|
||
|
|
catalog_rank = excluded.catalog_rank;
|
||
|
|
""",
|
||
|
|
(artist_id, hiffi_rank, 1, "Top 500"),
|
||
|
|
)
|
||
|
|
|
||
|
|
sales_units = clean_float(row.get("streaming_eas"), 0.0)
|
||
|
|
sales_rank = clean_int(row.get("cohort_sales_rank") or row.get("chartmasters_sales_rank"), 0)
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO commercial_sales (artist_id, audited_sales_units, provenance, sales_rank)
|
||
|
|
VALUES (?, ?, ?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO UPDATE SET
|
||
|
|
audited_sales_units = CASE WHEN excluded.audited_sales_units > 0.0 THEN excluded.audited_sales_units ELSE commercial_sales.audited_sales_units END,
|
||
|
|
sales_rank = CASE WHEN excluded.sales_rank > 0 THEN excluded.sales_rank ELSE commercial_sales.sales_rank END;
|
||
|
|
""",
|
||
|
|
(artist_id, sales_units, "Multiplatform Pipeline", sales_rank),
|
||
|
|
)
|
||
|
|
|
||
|
|
subs = clean_int(row.get("subscriber_count"), 0)
|
||
|
|
listeners = clean_int(row.get("spotify_monthly_listeners"), 0)
|
||
|
|
vel_7d = clean_float(row.get("daily_view_velocity"), 0.0)
|
||
|
|
vel_30d = clean_float(row.get("momentum_30d"), 0.0)
|
||
|
|
|
||
|
|
cursor.execute(
|
||
|
|
"""
|
||
|
|
INSERT INTO telemetry_metrics (
|
||
|
|
artist_id, subscribers_count, spotify_monthly_listeners,
|
||
|
|
growth_velocity_7d, growth_velocity_30d
|
||
|
|
)
|
||
|
|
VALUES (?, ?, ?, ?, ?)
|
||
|
|
ON CONFLICT(artist_id) DO UPDATE SET
|
||
|
|
subscribers_count = excluded.subscribers_count,
|
||
|
|
spotify_monthly_listeners = CASE WHEN excluded.spotify_monthly_listeners > 0 THEN excluded.spotify_monthly_listeners ELSE telemetry_metrics.spotify_monthly_listeners END,
|
||
|
|
growth_velocity_7d = excluded.growth_velocity_7d,
|
||
|
|
growth_velocity_30d = excluded.growth_velocity_30d;
|
||
|
|
""",
|
||
|
|
(artist_id, subs, listeners, vel_7d, vel_30d),
|
||
|
|
)
|
||
|
|
|
||
|
|
conn.commit()
|
||
|
|
print(f"Multiplatform 500 cohort ingested: {len(df_500)} records.")
|
||
|
|
|
||
|
|
conn.close()
|
||
|
|
print("Zero-null backfill completed successfully.")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
backfill()
|