hiffi-artist-intelligence/scripts/audit_and_scrape_real_faces.py

"""
scripts/audit_and_scrape_real_faces.py
Fail-safe face audit and scraper.
Forces immediate console output and scans every cohort image.
"""

import sys
import os
import sqlite3
import requests
import re
import csv
from pathlib import Path
from PIL import Image, ImageStat

print("\n" + "="*50, flush=True)
print("[*] Starting HIFFI Real Face Audit & Scraper...", flush=True)
print("="*50, flush=True)

# Absolute path resolution
BASE_DIR = Path(__file__).resolve().parent.parent
DB_PATH = BASE_DIR / "data" / "hiffi_catalog.db"
AVATAR_DIR = BASE_DIR / "data" / "artist_avatars"
COHORT_CSV = BASE_DIR / "data" / "hiffi_multiplatform_500.csv"
FALLBACK_CSV = BASE_DIR / "cleaned_artists_inventory.csv"

AVATAR_DIR.mkdir(parents=True, exist_ok=True)

print(f"[*] Base Directory: {BASE_DIR}", flush=True)
print(f"[*] Database Path:  {DB_PATH}", flush=True)
print(f"[*] Database Exists: {DB_PATH.exists()}", flush=True)

if not DB_PATH.exists():
    print(f"[-] ERROR: Database not found at {DB_PATH}", flush=True)
    sys.exit(1)

HEADERS = {
    "User-Agent": "HIFFIArtistIntelligence/1.0 (contact@hiffi.ai; https://hiffi.ai)",
    "Accept": "application/json,image/*"
}

def clean_file_slug(name: str) -> str:
    cleaned = re.sub(r'[^a-zA-Z0-9_-]', '_', name.strip().lower())
    return re.sub(r'_+', '_', cleaned).strip('_')

def load_cohort_artists() -> list:
    target_csv = COHORT_CSV if COHORT_CSV.exists() else FALLBACK_CSV
    names = []
    if target_csv.exists():
        with open(target_csv, "r", encoding="utf-8-sig") as f:
            reader = csv.DictReader(f)
            for row in reader:
                name = row.get("artist_name") or row.get("Artist") or row.get("name")
                if name and name.strip():
                    names.append(name.strip())
    return names

def is_suspect_image(image_path: Path):
    """Checks if an image is missing, tiny, or a flat logo/album graphic."""
    if not image_path.exists():
        return True, "File does not exist"
    
    file_size = image_path.stat().st_size
    if file_size < 3000:
        return True, f"File too small ({file_size} bytes)"

    try:
        with Image.open(image_path) as img:
            img = img.convert("RGB")
            if img.width < 120 or img.height < 120:
                return True, f"Low resolution ({img.width}x{img.height})"

            stat = ImageStat.Stat(img)
            avg_stddev = sum(stat.stddev) / len(stat.stddev)
            if avg_stddev < 18.0:
                return True, f"Flat graphic/text logo (stddev {avg_stddev:.1f})"
        return False, "OK"
    except Exception as e:
        return True, f"Corrupted image ({e})"

def search_wikipedia_portrait(artist_name: str) -> str:
    search_terms = [
        f"{artist_name} rapper",
        f"{artist_name} musician",
        artist_name
    ]
    for term in search_terms:
        api_url = (
            f"https://en.wikipedia.org/w/api.php?action=query&generator=search"
            f"&gsrsearch={requests.utils.quote(term)}&gsrlimit=1&prop=pageimages"
            f"&pithumbsize=600&format=json"
        )
        try:
            res = requests.get(api_url, headers=HEADERS, timeout=8).json()
            pages = res.get("query", {}).get("pages", {})
            for pid, pdata in pages.items():
                if "thumbnail" in pdata and pdata["thumbnail"].get("source"):
                    src = pdata["thumbnail"]["source"]
                    if not any(x in src.lower() for x in ["flag", "icon", "stub", "logo"]):
                        return src
        except Exception:
            continue
    return ""

def search_deezer_portrait(artist_name: str) -> str:
    clean_name = re.sub(r'[\$@#!\?]', '', artist_name).strip()
    api_url = f"https://api.deezer.com/search/artist?q={requests.utils.quote(clean_name)}&limit=3"
    try:
        res = requests.get(api_url, headers=HEADERS, timeout=8).json()
        if "data" in res and len(res["data"]) > 0:
            for item in res["data"]:
                pic = item.get("picture_big") or item.get("picture_medium")
                if pic and "artist//500x500" not in pic:
                    return pic
    except Exception:
        pass
    return ""

def main():
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()

    cohort_names = load_cohort_artists()
    print(f"[*] Loaded {len(cohort_names)} names from cohort CSV.", flush=True)

    if cohort_names:
        placeholders = ",".join("?" for _ in cohort_names)
        cursor.execute(f"SELECT artist_id, artist_name, image_url FROM artists WHERE artist_name IN ({placeholders})", cohort_names)
    else:
        cursor.execute("SELECT artist_id, artist_name, image_url FROM artists LIMIT 500")

    records = cursor.fetchall()
    print(f"[*] Found {len(records)} matching artist records in database.", flush=True)
    print("[*] Starting visual quality audit...\n", flush=True)

    suspect_artists = []

    for idx, r in enumerate(records, 1):
        artist_id = str(r["artist_id"])
        artist_name = r["artist_name"]
        slug = clean_file_slug(artist_name)

        local_file = None
        for ext in [".jpg", ".png", ".webp", ".jpeg"]:
            cand = AVATAR_DIR / f"{slug}_{artist_id}{ext}"
            if cand.exists():
                local_file = cand
                break

        is_bad, reason = is_suspect_image(local_file) if local_file else (True, "No local file")

        if is_bad:
            suspect_artists.append((artist_id, artist_name, reason))
            print(f"  [{idx}/500] [!] Flagged: {artist_name} -> Reason: {reason}", flush=True)

    print(f"\n[*] Audit Complete: {len(suspect_artists)} artists need verified portraits.", flush=True)

    if not suspect_artists:
        print("[+] All cohort images are high-quality face portraits!", flush=True)
        conn.close()
        return

    print("\n[*] Scraping verified editorial portraits...", flush=True)
    resolved_count = 0

    for artist_id, artist_name, reason in suspect_artists:
        print(f"  [>] Fetching portrait for: {artist_name}...", flush=True)
        portrait_url = search_wikipedia_portrait(artist_name)
        if not portrait_url:
            portrait_url = search_deezer_portrait(artist_name)

        if portrait_url:
            slug = clean_file_slug(artist_name)
            target_path = AVATAR_DIR / f"{slug}_{artist_id}.jpg"
            try:
                img_res = requests.get(portrait_url, headers=HEADERS, timeout=12)
                if img_res.status_code == 200 and len(img_res.content) > 2000:
                    with open(target_path, "wb") as f:
                        f.write(img_res.content)

                    cursor.execute(
                        "UPDATE artists SET image_url = ?, image_verified_flag = 1 WHERE artist_id = ?",
                        (portrait_url, artist_id)
                    )
                    conn.commit()
                    resolved_count += 1
                    print(f"      [+] Saved portrait: {target_path.name} ({len(img_res.content)//1024} KB)", flush=True)
                else:
                    print(f"      [-] Download failed for {artist_name}: HTTP {img_res.status_code}", flush=True)
            except Exception as e:
                print(f"      [-] Download error for {artist_name}: {e}", flush=True)
        else:
            print(f"      [-] No portrait found for {artist_name}", flush=True)

    conn.close()
    print(f"\n[+] Complete! Resolved {resolved_count}/{len(suspect_artists)} portraits.", flush=True)

if __name__ == "__main__":
    main()