feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort
This commit is contained in:
Srilekha committed
1 parent
85e6395770
commit
78a46a8dbe
629 files changed
+123965
-755
No files matched your search
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
scripts/backfill_catalog_data.py
|
||||
Dual-source ingestion engine for master catalog and multiplatform telemetry.
|
||||
Enforces the Zero-Null Standard across all numeric and velocity fields.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parent.parent
|
||||
if str(ROOT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT_DIR))
|
||||
|
||||
from core.database import get_connection
|
||||
|
||||
MASTER_PATH = ROOT_DIR / "data" / "master_artists_intelligence_17k_final.csv"
|
||||
MULTIPLATFORM_PATH = ROOT_DIR / "data" / "hiffi_multiplatform_500.csv"
|
||||
|
||||
|
||||
def clean_str(val, default=""):
|
||||
if pd.isna(val) or val is None:
|
||||
return default
|
||||
s = str(val).strip()
|
||||
return default if s.lower() in ("nan", "none", "null") else s
|
||||
|
||||
|
||||
def clean_int(val, default=0):
|
||||
if pd.isna(val) or val is None:
|
||||
return default
|
||||
try:
|
||||
return int(float(val))
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def clean_float(val, default=0.0):
|
||||
if pd.isna(val) or val is None:
|
||||
return default
|
||||
try:
|
||||
f = float(val)
|
||||
return default if np.isnan(f) or np.isinf(f) else f
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def generate_slug_id(name: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9]+", "_", name.strip().lower()).strip("_")
|
||||
return f"art_{slug}" if slug else "art_unknown"
|
||||
|
||||
|
||||
def backfill():
|
||||
conn = get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
print("Beginning master catalog ingestion (17k)...")
|
||||
name_to_id = {}
|
||||
|
||||
if MASTER_PATH.exists():
|
||||
df_master = pd.read_csv(MASTER_PATH, low_memory=False)
|
||||
for _, row in df_master.iterrows():
|
||||
artist_id = clean_str(row.get("artist_id"))
|
||||
artist_name = clean_str(row.get("artist_name"))
|
||||
if not artist_id or not artist_name:
|
||||
continue
|
||||
|
||||
name_to_id[artist_name.lower()] = artist_id
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO artists (artist_id, artist_name, country_origin)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
artist_name = excluded.artist_name;
|
||||
""",
|
||||
(artist_id, artist_name, "Unknown"),
|
||||
)
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO rankings (artist_id, catalog_rank, tier_rank, catalog_intelligence_tier)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
catalog_rank = excluded.catalog_rank,
|
||||
tier_rank = excluded.tier_rank,
|
||||
catalog_intelligence_tier = excluded.catalog_intelligence_tier;
|
||||
""",
|
||||
(
|
||||
artist_id,
|
||||
clean_int(row.get("catalog_rank"), 99999),
|
||||
clean_int(row.get("tier_rank"), 0),
|
||||
clean_str(row.get("catalog_intelligence_tier"), "Emerging"),
|
||||
),
|
||||
)
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO commercial_sales (artist_id, audited_sales_units, awards_count, provenance, sales_rank)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
audited_sales_units = excluded.audited_sales_units,
|
||||
awards_count = excluded.awards_count,
|
||||
provenance = excluded.provenance,
|
||||
sales_rank = excluded.sales_rank;
|
||||
""",
|
||||
(
|
||||
artist_id,
|
||||
clean_float(row.get("audited_sales_units"), 0.0),
|
||||
clean_int(row.get("awards_count"), 0),
|
||||
clean_str(row.get("provenance"), "N/A"),
|
||||
clean_int(row.get("sales_rank"), 0),
|
||||
),
|
||||
)
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO telemetry_metrics (
|
||||
artist_id, subscribers_count, spotify_monthly_listeners,
|
||||
growth_velocity_7d, growth_velocity_30d
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(artist_id) DO NOTHING;
|
||||
""",
|
||||
(
|
||||
artist_id,
|
||||
0,
|
||||
clean_int(row.get("spotify_monthly_listeners"), 0),
|
||||
0.0,
|
||||
0.0,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
print(f"Master catalog processed: {len(df_master)} records.")
|
||||
|
||||
print("Processing multiplatform 500 cohort...")
|
||||
if MULTIPLATFORM_PATH.exists():
|
||||
df_500 = pd.read_csv(MULTIPLATFORM_PATH)
|
||||
for _, row in df_500.iterrows():
|
||||
name = clean_str(row.get("name"))
|
||||
if not name:
|
||||
continue
|
||||
|
||||
artist_id = name_to_id.get(name.lower())
|
||||
if not artist_id:
|
||||
raw_ident = clean_str(row.get("identifier"))
|
||||
artist_id = raw_ident if raw_ident else generate_slug_id(name)
|
||||
name_to_id[name.lower()] = artist_id
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO artists (artist_id, artist_name)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
artist_name = excluded.artist_name;
|
||||
""",
|
||||
(artist_id, name),
|
||||
)
|
||||
|
||||
hiffi_rank = clean_int(
|
||||
row.get("hiffi_xp_published_rank") or row.get("production_hiffi_rank") or row.get("hiffi_rank"),
|
||||
99999,
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO rankings (artist_id, catalog_rank, tier_rank, catalog_intelligence_tier)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
catalog_rank = excluded.catalog_rank;
|
||||
""",
|
||||
(artist_id, hiffi_rank, 1, "Top 500"),
|
||||
)
|
||||
|
||||
sales_units = clean_float(row.get("streaming_eas"), 0.0)
|
||||
sales_rank = clean_int(row.get("cohort_sales_rank") or row.get("chartmasters_sales_rank"), 0)
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO commercial_sales (artist_id, audited_sales_units, provenance, sales_rank)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
audited_sales_units = CASE WHEN excluded.audited_sales_units > 0.0 THEN excluded.audited_sales_units ELSE commercial_sales.audited_sales_units END,
|
||||
sales_rank = CASE WHEN excluded.sales_rank > 0 THEN excluded.sales_rank ELSE commercial_sales.sales_rank END;
|
||||
""",
|
||||
(artist_id, sales_units, "Multiplatform Pipeline", sales_rank),
|
||||
)
|
||||
|
||||
subs = clean_int(row.get("subscriber_count"), 0)
|
||||
listeners = clean_int(row.get("spotify_monthly_listeners"), 0)
|
||||
vel_7d = clean_float(row.get("daily_view_velocity"), 0.0)
|
||||
vel_30d = clean_float(row.get("momentum_30d"), 0.0)
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO telemetry_metrics (
|
||||
artist_id, subscribers_count, spotify_monthly_listeners,
|
||||
growth_velocity_7d, growth_velocity_30d
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
subscribers_count = excluded.subscribers_count,
|
||||
spotify_monthly_listeners = CASE WHEN excluded.spotify_monthly_listeners > 0 THEN excluded.spotify_monthly_listeners ELSE telemetry_metrics.spotify_monthly_listeners END,
|
||||
growth_velocity_7d = excluded.growth_velocity_7d,
|
||||
growth_velocity_30d = excluded.growth_velocity_30d;
|
||||
""",
|
||||
(artist_id, subs, listeners, vel_7d, vel_30d),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
print(f"Multiplatform 500 cohort ingested: {len(df_500)} records.")
|
||||
|
||||
conn.close()
|
||||
print("Zero-null backfill completed successfully.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
backfill()
|
||||
Reference in new issue
Block a user