feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort
This commit is contained in:
Srilekha committed
1 parent
85e6395770
commit
78a46a8dbe
629 files changed
+123965
-755
No files matched your search
@@ -0,0 +1,178 @@
|
||||
"""
|
||||
scripts/replace_non_face_avatars.py
|
||||
Uses Apple iTunes & MusicBrainz API to find authentic artist portraits
|
||||
for any cohort member currently using an album cover, logo, or placeholder.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import requests
|
||||
import re
|
||||
import csv
|
||||
import time
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageStat
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
DB_PATH = BASE_DIR / "data" / "hiffi_catalog.db"
|
||||
AVATAR_DIR = BASE_DIR / "data" / "artist_avatars"
|
||||
COHORT_CSV = BASE_DIR / "data" / "hiffi_multiplatform_500.csv"
|
||||
FALLBACK_CSV = BASE_DIR / "cleaned_artists_inventory.csv"
|
||||
|
||||
AVATAR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
def clean_file_slug(name: str) -> str:
|
||||
cleaned = re.sub(r'[^a-zA-Z0-9_-]', '_', name.strip().lower())
|
||||
return re.sub(r'_+', '_', cleaned).strip('_')
|
||||
|
||||
def load_cohort():
|
||||
target = COHORT_CSV if COHORT_CSV.exists() else FALLBACK_CSV
|
||||
artists = []
|
||||
if target.exists():
|
||||
with open(target, "r", encoding="utf-8-sig") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
name = row.get("artist_name") or row.get("Artist") or row.get("name")
|
||||
if name and name.strip():
|
||||
artists.append(name.strip())
|
||||
return artists
|
||||
|
||||
def is_bad_or_nonface_image(path: Path) -> bool:
|
||||
"""Detects missing, silhouette, or low-contrast graphics."""
|
||||
if not path.exists():
|
||||
return True
|
||||
if path.stat().st_size < 3500:
|
||||
return True
|
||||
try:
|
||||
with Image.open(path) as img:
|
||||
img = img.convert("RGB")
|
||||
stat = ImageStat.Stat(img)
|
||||
avg_stddev = sum(stat.stddev) / len(stat.stddev)
|
||||
# Extremely flat images (like the gray silhouette) have stddev < 18
|
||||
if avg_stddev < 18.0:
|
||||
return True
|
||||
return False
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
def search_itunes_artist_image(artist_name: str) -> str:
|
||||
"""Searches iTunes for the artist's high-res editorial image."""
|
||||
clean_name = re.sub(r'[\$@#!\?]', '', artist_name).strip()
|
||||
url = f"https://itunes.apple.com/search?term={requests.utils.quote(clean_name)}&entity=musicArtist&limit=1"
|
||||
try:
|
||||
res = requests.get(url, headers=HEADERS, timeout=8).json()
|
||||
if res.get("resultCount", 0) > 0:
|
||||
artist_id = res["results"][0].get("artistId")
|
||||
if artist_id:
|
||||
# Look up artist lookups for artwork
|
||||
lookup_url = f"https://itunes.apple.com/lookup?id={artist_id}&entity=album&limit=1"
|
||||
l_res = requests.get(lookup_url, headers=HEADERS, timeout=8).json()
|
||||
if l_res.get("resultCount", 0) > 1:
|
||||
art = l_res["results"][1].get("artworkUrl100")
|
||||
if art:
|
||||
# Request the uncompressed 1000x1000 version
|
||||
return art.replace("100x100bb.jpg", "1000x1000bb.jpg")
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
def search_wikipedia_face(artist_name: str) -> str:
|
||||
"""Searches Wikipedia API for the direct lead photo thumbnail."""
|
||||
clean_name = re.sub(r'[\$@#!\?]', '', artist_name).strip()
|
||||
wiki_headers = {"User-Agent": "HIFFIArtistApp/1.0 (contact@hiffi.ai)"}
|
||||
for q in [f"{clean_name} rapper", f"{clean_name} musician", clean_name]:
|
||||
url = f"https://en.wikipedia.org/w/api.php?action=query&generator=search&gsrsearch={requests.utils.quote(q)}&gsrlimit=1&prop=pageimages&pithumbsize=600&format=json"
|
||||
try:
|
||||
r = requests.get(url, headers=wiki_headers, timeout=8).json()
|
||||
pages = r.get("query", {}).get("pages", {})
|
||||
for pid, pdata in pages.items():
|
||||
if "thumbnail" in pdata and pdata["thumbnail"].get("source"):
|
||||
src = pdata["thumbnail"]["source"]
|
||||
if not any(x in src.lower() for x in ["flag", "logo", "icon", "stub"]):
|
||||
return src
|
||||
except Exception:
|
||||
continue
|
||||
return ""
|
||||
|
||||
def main():
|
||||
if not DB_PATH.exists():
|
||||
print(f"[-] Database not found at {DB_PATH}")
|
||||
return
|
||||
|
||||
cohort = load_cohort()
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
if cohort:
|
||||
placeholders = ",".join("?" for _ in cohort)
|
||||
cursor.execute(f"SELECT artist_id, artist_name, image_url FROM artists WHERE artist_name IN ({placeholders})", cohort)
|
||||
else:
|
||||
cursor.execute("SELECT artist_id, artist_name, image_url FROM artists LIMIT 500")
|
||||
|
||||
records = cursor.fetchall()
|
||||
print(f"[*] Scanning {len(records)} artists for bad/non-face profile images...\n")
|
||||
|
||||
flagged = []
|
||||
for r in records:
|
||||
artist_id = str(r["artist_id"])
|
||||
name = r["artist_name"]
|
||||
slug = clean_file_slug(name)
|
||||
|
||||
local_file = None
|
||||
for ext in [".jpg", ".png", ".webp", ".jpeg"]:
|
||||
cand = AVATAR_DIR / f"{slug}_{artist_id}{ext}"
|
||||
if cand.exists():
|
||||
local_file = cand
|
||||
break
|
||||
|
||||
if not local_file or is_bad_or_nonface_image(local_file):
|
||||
flagged.append((artist_id, name))
|
||||
|
||||
print(f"[*] Total artists needing face portraits: {len(flagged)}")
|
||||
|
||||
if not flagged:
|
||||
print("[+] All 500 artists have confirmed face portraits!")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
print("\n[*] Fetching authentic face portraits from Wikipedia & Apple Music...")
|
||||
resolved = 0
|
||||
|
||||
for artist_id, name in flagged:
|
||||
slug = clean_file_slug(name)
|
||||
target_path = AVATAR_DIR / f"{slug}_{artist_id}.jpg"
|
||||
print(f" [>] Finding portrait for: {name}...", end=" ", flush=True)
|
||||
|
||||
new_url = search_wikipedia_face(name)
|
||||
if not new_url:
|
||||
new_url = search_itunes_artist_image(name)
|
||||
|
||||
if new_url:
|
||||
try:
|
||||
res = requests.get(new_url, headers=HEADERS, timeout=12)
|
||||
if res.status_code == 200 and len(res.content) > 2500:
|
||||
with open(target_path, "wb") as f:
|
||||
f.write(res.content)
|
||||
|
||||
cursor.execute(
|
||||
"UPDATE artists SET image_url = ?, image_verified_flag = 1 WHERE artist_id = ?",
|
||||
(new_url, artist_id)
|
||||
)
|
||||
conn.commit()
|
||||
resolved += 1
|
||||
print(f"SAVED ({len(res.content)//1024} KB)")
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print("FAILED")
|
||||
|
||||
conn.close()
|
||||
print(f"\n[+] Finished. Resolved {resolved}/{len(flagged)} portraits.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user