112 lines
4.1 KiB
Python
112 lines
4.1 KiB
Python
"""
|
|
scripts/download_verified_portraits_local.py
|
|
Downloads verified artist images locally to data/artist_avatars/.
|
|
Uses browser headers to prevent CDN 403 Forbidden errors.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import sqlite3
|
|
import requests
|
|
import re
|
|
from pathlib import Path
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
|
DB_PATH = BASE_DIR / "data" / "hiffi_catalog.db"
|
|
AVATAR_DIR = BASE_DIR / "data" / "artist_avatars"
|
|
|
|
# Ensure the local avatar storage folder exists
|
|
AVATAR_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Headers that tell CDNs this is a standard web browser request
|
|
DEFAULT_HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
"Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
"Referer": "https://www.deezer.com/"
|
|
}
|
|
|
|
def clean_file_slug(name: str) -> str:
|
|
"""Turn artist names into safe file names (e.g. '$uicideboy$' -> '_uicideboy_')."""
|
|
cleaned = re.sub(r'[^a-zA-Z0-9_-]', '_', name.strip().lower())
|
|
return re.sub(r'_+', '_', cleaned).strip('_')
|
|
|
|
def download_single_artist(artist_id: str, artist_name: str, url: str):
|
|
"""Downloads one artist portrait and saves it locally."""
|
|
if not url or url.startswith("data:image"):
|
|
return artist_name, False, "Empty or invalid URL"
|
|
|
|
slug = clean_file_slug(artist_name)
|
|
ext = ".jpg"
|
|
if ".png" in url.lower():
|
|
ext = ".png"
|
|
elif ".webp" in url.lower():
|
|
ext = ".webp"
|
|
|
|
local_filename = f"{slug}_{artist_id}{ext}"
|
|
local_path = AVATAR_DIR / local_filename
|
|
|
|
# If the file already exists and is not corrupt/empty (larger than 2 KB), reuse it
|
|
if local_path.exists() and local_path.stat().st_size > 2048:
|
|
return artist_name, True, "Already cached"
|
|
|
|
# Set headers depending on the host
|
|
headers = DEFAULT_HEADERS.copy()
|
|
if "wikimedia.org" in url or "wikipedia.org" in url:
|
|
headers["Referer"] = "https://en.wikipedia.org/"
|
|
headers["User-Agent"] = "HIFFIArtistIntelligence/1.0 (contact@hiffi.ai)"
|
|
|
|
try:
|
|
response = requests.get(url, headers=headers, timeout=15)
|
|
if response.status_code == 200:
|
|
if len(response.content) < 1500:
|
|
return artist_name, False, "Response too small to be an image"
|
|
|
|
with open(local_path, "wb") as f:
|
|
f.write(response.content)
|
|
return artist_name, True, "Downloaded successfully"
|
|
else:
|
|
return artist_name, False, f"Server returned HTTP {response.status_code}"
|
|
except Exception as exc:
|
|
return artist_name, False, str(exc)
|
|
|
|
def main():
|
|
if not DB_PATH.exists():
|
|
print(f"Error: Database not found at {DB_PATH}")
|
|
sys.exit(1)
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT artist_id, artist_name, image_url FROM artists WHERE image_url IS NOT NULL AND image_url != ''")
|
|
records = cursor.fetchall()
|
|
conn.close()
|
|
|
|
print(f"[*] Starting local image download for {len(records)} artists...")
|
|
|
|
successful = 0
|
|
failures = []
|
|
|
|
# Download multiple images at once using 8 parallel workers for speed
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
future_to_artist = {
|
|
executor.submit(download_single_artist, row[0], row[1], row[2]): (row[1], row[2])
|
|
for row in records
|
|
}
|
|
|
|
for future in as_completed(future_to_artist):
|
|
name, url = future_to_artist[future]
|
|
artist_name, success, message = future.result()
|
|
if success:
|
|
successful += 1
|
|
else:
|
|
failures.append((artist_name, message, url))
|
|
|
|
print(f"\n[+] Local Caching Complete: {successful}/{len(records)} saved to {AVATAR_DIR}")
|
|
if failures:
|
|
print(f"[-] Remaining failed images ({len(failures)}):")
|
|
for fail_name, fail_msg, fail_url in failures[:10]:
|
|
print(f" - {fail_name}: {fail_msg} ({fail_url[:60]}...)")
|
|
|
|
if __name__ == "__main__":
|
|
main() |