checkpoint: working 10-artist batch with log-normalized ranking

verifiers/wikidata_verifier.py

@@ -0,0 +1,105 @@
+import os
+import json
+import sqlite3
+import requests
+from typing import Dict, Any
+from rapidfuzz import fuzz
+
+class WikidataVerifier:
+    """
+    Checks if an artist has a Wikidata entity with local SQLite caching.
+    """
+    SEARCH_URL = "https://www.wikidata.org/w/api.php"
+
+    def __init__(self, db_path: str = ".cache/verifiers_cache.sqlite"):
+        self.headers = {
+            "User-Agent": "HiffiArtistIntelligence/1.0 (contact@superlabs.co)"
+        }
+        self.db_path = db_path
+        os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
+        self._init_db()
+
+    def _init_db(self):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS wikidata_cache (
+                    artist_name TEXT PRIMARY KEY,
+                    data_json TEXT,
+                    cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """)
+
+    def _get_cache(self, name: str) -> Dict[str, Any]:
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT data_json FROM wikidata_cache WHERE artist_name = ?", (name.lower(),))
+            row = cursor.fetchone()
+            if row:
+                return json.loads(row[0])
+        return None
+
+    def _set_cache(self, name: str, data: Dict[str, Any]):
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO wikidata_cache (artist_name, data_json) VALUES (?, ?)",
+                (name.lower(), json.dumps(data))
+            )
+
+    def verify_artist(self, name: str) -> Dict[str, Any]:
+        if not name or not name.strip():
+            return {"verified": False, "confidence": 0.0, "reason": "empty_name"}
+
+        cleaned_name = name.strip()
+
+        # Check local cache first
+        cached = self._get_cache(cleaned_name)
+        if cached:
+            return cached
+
+        params = {
+            "action": "wbsearchentities",
+            "format": "json",
+            "language": "en",
+            "type": "item",
+            "search": cleaned_name,
+            "limit": 3
+        }
+
+        try:
+            resp = requests.get(self.SEARCH_URL, headers=self.headers, params=params, timeout=10)
+            if resp.status_code != 200:
+                result = {"verified": False, "confidence": 0.0, "status_code": resp.status_code}
+                return result
+
+            results = resp.json().get("search", [])
+            if not results:
+                result = {"verified": False, "confidence": 0.0, "reason": "not_found"}
+                self._set_cache(cleaned_name, result)
+                return result
+
+            best = results[0]
+            label = best.get("label", "")
+            desc = best.get("description", "").lower()
+            
+            music_keywords = ["musician", "singer", "rapper", "band", "record producer", "songwriter", "artist", "group"]
+            is_music_entity = any(k in desc for k in music_keywords)
+
+            similarity = fuzz.token_sort_ratio(cleaned_name.lower(), label.lower()) / 100.0
+            confidence = round(similarity * (1.0 if is_music_entity else 0.5), 2)
+
+            result = {
+                "verified": confidence >= 0.75,
+                "confidence": confidence,
+                "wikidata_id": best.get("id"),
+                "label": label,
+                "description": best.get("description", "")
+            }
+            self._set_cache(cleaned_name, result)
+            return result
+        except Exception as e:
+            return {"verified": False, "confidence": 0.0, "error": str(e)}
+
+if __name__ == "__main__":
+    verifier = WikidataVerifier()
+    print("Testing Wikidata Caching...")
+    print(verifier.verify_artist("Kendrick Lamar"))