feat: finalize root api.py with X-Auth-Secret auth security and clean pure sales cohort
This commit is contained in:
Srilekha committed
1 parent
85e6395770
commit
78a46a8dbe
629 files changed
+123965
-755
No files matched your search
@@ -0,0 +1,200 @@
|
||||
"""
|
||||
scripts/audit_and_scrape_real_faces.py
|
||||
Fail-safe face audit and scraper.
|
||||
Forces immediate console output and scans every cohort image.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import sqlite3
|
||||
import requests
|
||||
import re
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageStat
|
||||
|
||||
print("\n" + "="*50, flush=True)
|
||||
print("[*] Starting HIFFI Real Face Audit & Scraper...", flush=True)
|
||||
print("="*50, flush=True)
|
||||
|
||||
# Absolute path resolution
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
DB_PATH = BASE_DIR / "data" / "hiffi_catalog.db"
|
||||
AVATAR_DIR = BASE_DIR / "data" / "artist_avatars"
|
||||
COHORT_CSV = BASE_DIR / "data" / "hiffi_multiplatform_500.csv"
|
||||
FALLBACK_CSV = BASE_DIR / "cleaned_artists_inventory.csv"
|
||||
|
||||
AVATAR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"[*] Base Directory: {BASE_DIR}", flush=True)
|
||||
print(f"[*] Database Path: {DB_PATH}", flush=True)
|
||||
print(f"[*] Database Exists: {DB_PATH.exists()}", flush=True)
|
||||
|
||||
if not DB_PATH.exists():
|
||||
print(f"[-] ERROR: Database not found at {DB_PATH}", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "HIFFIArtistIntelligence/1.0 (contact@hiffi.ai; https://hiffi.ai)",
|
||||
"Accept": "application/json,image/*"
|
||||
}
|
||||
|
||||
def clean_file_slug(name: str) -> str:
|
||||
cleaned = re.sub(r'[^a-zA-Z0-9_-]', '_', name.strip().lower())
|
||||
return re.sub(r'_+', '_', cleaned).strip('_')
|
||||
|
||||
def load_cohort_artists() -> list:
|
||||
target_csv = COHORT_CSV if COHORT_CSV.exists() else FALLBACK_CSV
|
||||
names = []
|
||||
if target_csv.exists():
|
||||
with open(target_csv, "r", encoding="utf-8-sig") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
name = row.get("artist_name") or row.get("Artist") or row.get("name")
|
||||
if name and name.strip():
|
||||
names.append(name.strip())
|
||||
return names
|
||||
|
||||
def is_suspect_image(image_path: Path):
|
||||
"""Checks if an image is missing, tiny, or a flat logo/album graphic."""
|
||||
if not image_path.exists():
|
||||
return True, "File does not exist"
|
||||
|
||||
file_size = image_path.stat().st_size
|
||||
if file_size < 3000:
|
||||
return True, f"File too small ({file_size} bytes)"
|
||||
|
||||
try:
|
||||
with Image.open(image_path) as img:
|
||||
img = img.convert("RGB")
|
||||
if img.width < 120 or img.height < 120:
|
||||
return True, f"Low resolution ({img.width}x{img.height})"
|
||||
|
||||
stat = ImageStat.Stat(img)
|
||||
avg_stddev = sum(stat.stddev) / len(stat.stddev)
|
||||
if avg_stddev < 18.0:
|
||||
return True, f"Flat graphic/text logo (stddev {avg_stddev:.1f})"
|
||||
return False, "OK"
|
||||
except Exception as e:
|
||||
return True, f"Corrupted image ({e})"
|
||||
|
||||
def search_wikipedia_portrait(artist_name: str) -> str:
|
||||
search_terms = [
|
||||
f"{artist_name} rapper",
|
||||
f"{artist_name} musician",
|
||||
artist_name
|
||||
]
|
||||
for term in search_terms:
|
||||
api_url = (
|
||||
f"https://en.wikipedia.org/w/api.php?action=query&generator=search"
|
||||
f"&gsrsearch={requests.utils.quote(term)}&gsrlimit=1&prop=pageimages"
|
||||
f"&pithumbsize=600&format=json"
|
||||
)
|
||||
try:
|
||||
res = requests.get(api_url, headers=HEADERS, timeout=8).json()
|
||||
pages = res.get("query", {}).get("pages", {})
|
||||
for pid, pdata in pages.items():
|
||||
if "thumbnail" in pdata and pdata["thumbnail"].get("source"):
|
||||
src = pdata["thumbnail"]["source"]
|
||||
if not any(x in src.lower() for x in ["flag", "icon", "stub", "logo"]):
|
||||
return src
|
||||
except Exception:
|
||||
continue
|
||||
return ""
|
||||
|
||||
def search_deezer_portrait(artist_name: str) -> str:
|
||||
clean_name = re.sub(r'[\$@#!\?]', '', artist_name).strip()
|
||||
api_url = f"https://api.deezer.com/search/artist?q={requests.utils.quote(clean_name)}&limit=3"
|
||||
try:
|
||||
res = requests.get(api_url, headers=HEADERS, timeout=8).json()
|
||||
if "data" in res and len(res["data"]) > 0:
|
||||
for item in res["data"]:
|
||||
pic = item.get("picture_big") or item.get("picture_medium")
|
||||
if pic and "artist//500x500" not in pic:
|
||||
return pic
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
def main():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
cohort_names = load_cohort_artists()
|
||||
print(f"[*] Loaded {len(cohort_names)} names from cohort CSV.", flush=True)
|
||||
|
||||
if cohort_names:
|
||||
placeholders = ",".join("?" for _ in cohort_names)
|
||||
cursor.execute(f"SELECT artist_id, artist_name, image_url FROM artists WHERE artist_name IN ({placeholders})", cohort_names)
|
||||
else:
|
||||
cursor.execute("SELECT artist_id, artist_name, image_url FROM artists LIMIT 500")
|
||||
|
||||
records = cursor.fetchall()
|
||||
print(f"[*] Found {len(records)} matching artist records in database.", flush=True)
|
||||
print("[*] Starting visual quality audit...\n", flush=True)
|
||||
|
||||
suspect_artists = []
|
||||
|
||||
for idx, r in enumerate(records, 1):
|
||||
artist_id = str(r["artist_id"])
|
||||
artist_name = r["artist_name"]
|
||||
slug = clean_file_slug(artist_name)
|
||||
|
||||
local_file = None
|
||||
for ext in [".jpg", ".png", ".webp", ".jpeg"]:
|
||||
cand = AVATAR_DIR / f"{slug}_{artist_id}{ext}"
|
||||
if cand.exists():
|
||||
local_file = cand
|
||||
break
|
||||
|
||||
is_bad, reason = is_suspect_image(local_file) if local_file else (True, "No local file")
|
||||
|
||||
if is_bad:
|
||||
suspect_artists.append((artist_id, artist_name, reason))
|
||||
print(f" [{idx}/500] [!] Flagged: {artist_name} -> Reason: {reason}", flush=True)
|
||||
|
||||
print(f"\n[*] Audit Complete: {len(suspect_artists)} artists need verified portraits.", flush=True)
|
||||
|
||||
if not suspect_artists:
|
||||
print("[+] All cohort images are high-quality face portraits!", flush=True)
|
||||
conn.close()
|
||||
return
|
||||
|
||||
print("\n[*] Scraping verified editorial portraits...", flush=True)
|
||||
resolved_count = 0
|
||||
|
||||
for artist_id, artist_name, reason in suspect_artists:
|
||||
print(f" [>] Fetching portrait for: {artist_name}...", flush=True)
|
||||
portrait_url = search_wikipedia_portrait(artist_name)
|
||||
if not portrait_url:
|
||||
portrait_url = search_deezer_portrait(artist_name)
|
||||
|
||||
if portrait_url:
|
||||
slug = clean_file_slug(artist_name)
|
||||
target_path = AVATAR_DIR / f"{slug}_{artist_id}.jpg"
|
||||
try:
|
||||
img_res = requests.get(portrait_url, headers=HEADERS, timeout=12)
|
||||
if img_res.status_code == 200 and len(img_res.content) > 2000:
|
||||
with open(target_path, "wb") as f:
|
||||
f.write(img_res.content)
|
||||
|
||||
cursor.execute(
|
||||
"UPDATE artists SET image_url = ?, image_verified_flag = 1 WHERE artist_id = ?",
|
||||
(portrait_url, artist_id)
|
||||
)
|
||||
conn.commit()
|
||||
resolved_count += 1
|
||||
print(f" [+] Saved portrait: {target_path.name} ({len(img_res.content)//1024} KB)", flush=True)
|
||||
else:
|
||||
print(f" [-] Download failed for {artist_name}: HTTP {img_res.status_code}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" [-] Download error for {artist_name}: {e}", flush=True)
|
||||
else:
|
||||
print(f" [-] No portrait found for {artist_name}", flush=True)
|
||||
|
||||
conn.close()
|
||||
print(f"\n[+] Complete! Resolved {resolved_count}/{len(suspect_artists)} portraits.", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user