hiffi-artist-intelligence/scripts/export_missing_faces_list.py

"""
scripts/export_missing_faces_list.py
Finds every artist without a confirmed real face photo.
Prints the numbered list in terminal and saves data/ordered_artists_to_fix.csv.
"""

import sqlite3
import csv
import re
import math
from pathlib import Path
from PIL import Image, ImageStat, ImageFilter

BASE_DIR = Path(__file__).resolve().parent.parent
DB_PATH = BASE_DIR / "data" / "hiffi_catalog.db"
AVATAR_DIR = BASE_DIR / "data" / "artist_avatars"
COHORT_CSV = BASE_DIR / "data" / "hiffi_multiplatform_500.csv"
FALLBACK_CSV = BASE_DIR / "cleaned_artists_inventory.csv"
OUTPUT_FILE = BASE_DIR / "data" / "ordered_artists_to_fix.csv"

def clean_file_slug(name: str) -> str:
    cleaned = re.sub(r'[^a-zA-Z0-9_-]', '_', name.strip().lower())
    return re.sub(r'_+', '_', cleaned).strip('_')

def load_cohort_names() -> list:
    target_csv = COHORT_CSV if COHORT_CSV.exists() else FALLBACK_CSV
    names = []
    if target_csv.exists():
        with open(target_csv, "r", encoding="utf-8-sig") as f:
            reader = csv.DictReader(f)
            for row in reader:
                name = row.get("artist_name") or row.get("Artist") or row.get("name")
                if name and name.strip():
                    names.append(name.strip())
    return names

def calculate_entropy(img: Image.Image) -> float:
    histogram = img.histogram()
    total = sum(histogram)
    if total == 0:
        return 0.0
    entropy = 0.0
    for count in histogram:
        if count > 0:
            p = count / total
            entropy -= p * math.log2(p)
    return entropy

def is_real_human_face(image_path: Path):
    if not image_path.exists() or image_path.stat().st_size < 3000:
        return False, "Image file missing or empty"

    try:
        with Image.open(image_path) as img:
            img_rgb = img.convert("RGB")
            stat = ImageStat.Stat(img_rgb)
            avg_stddev = sum(stat.stddev) / len(stat.stddev)
            
            # Flat graphics or black background signatures
            if avg_stddev < 18.0:
                return False, "Signature or flat background graphic"

            # Drawings and cartoon artwork
            entropy = calculate_entropy(img_rgb)
            if entropy < 5.8:
                return False, "Cartoon, vector art, or drawing"

            return True, "Valid Photo"
    except Exception:
        return False, "Cannot read file"

def main():
    if not DB_PATH.exists():
        print(f"[-] Database not found at {DB_PATH}")
        return

    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()

    cohort_names = load_cohort_names()
    if cohort_names:
        placeholders = ",".join("?" for _ in cohort_names)
        cursor.execute(f"SELECT artist_id, artist_name FROM artists WHERE artist_name IN ({placeholders}) ORDER BY artist_name COLLATE NOCASE ASC", cohort_names)
    else:
        cursor.execute("SELECT artist_id, artist_name FROM artists ORDER BY artist_name COLLATE NOCASE ASC LIMIT 500")

    records = cursor.fetchall()
    conn.close()

    missing_faces = []

    for r in records:
        artist_id = str(r["artist_id"])
        artist_name = r["artist_name"]
        slug = clean_file_slug(artist_name)

        local_file = None
        for ext in [".jpg", ".png", ".webp", ".jpeg"]:
            cand = AVATAR_DIR / f"{slug}_{artist_id}{ext}"
            if cand.exists():
                local_file = cand
                break

        is_face, reason = is_real_human_face(local_file) if local_file else (False, "Missing file")

        if not is_face:
            missing_faces.append({
                "artist_id": artist_id,
                "artist_name": artist_name,
                "reason": reason
            })

    # Save to CSV
    with open(OUTPUT_FILE, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["order_number", "artist_id", "artist_name", "reason"])
        writer.writeheader()
        for idx, item in enumerate(missing_faces, 1):
            writer.writerow({
                "order_number": idx,
                "artist_id": item["artist_id"],
                "artist_name": item["artist_name"],
                "reason": item["reason"]
            })

    print("\n" + "="*60)
    print(f"  ARTISTS NEEDING REAL FACES (TOTAL: {len(missing_faces)})")
    print("="*60)
    for idx, item in enumerate(missing_faces, 1):
        print(f"  #{idx:02d} | {item['artist_name']} (Reason: {item['reason']})")
    print("="*60)
    print(f"[+] Ordered list exported to: {OUTPUT_FILE}")

if __name__ == "__main__":
    main()