feat(pipeline): finalize audited multi-platform dataset with clean provenance and entity resolution

collectors/wikidata_resolver.py

@@ -0,0 +1,113 @@
+"""
+collectors/wikidata_resolver.py
+Directly resolves canonical Wikidata Q-IDs, official Spotify Artist IDs (P1902),
+and Instagram usernames (P2003) via Wikidata's public entity API.
+"""
+
+import requests
+import pandas as pd
+from typing import List, Dict, Any
+
+WIKIDATA_SEARCH_URL = "https://www.wikidata.org/w/api.php"
+HEADERS = {
+    "User-Agent": "HIFFIArtistIntelligence/1.0 (sri@superlabs.ai) python-requests"
+}
+
+
+def search_wikidata_qid(artist_name: str) -> str:
+    """Finds the most relevant Wikidata Q-identifier for a music artist."""
+    params = {
+        "action": "wbsearchentities",
+        "format": "json",
+        "language": "en",
+        "type": "item",
+        "search": artist_name,
+        "limit": 5,
+    }
+    try:
+        res = requests.get(WIKIDATA_SEARCH_URL, params=params, headers=HEADERS, timeout=10)
+        if res.status_code == 200:
+            data = res.json().get("search", [])
+            for item in data:
+                desc = item.get("description", "").lower()
+                if any(k in desc for k in ["rapper", "singer", "musician", "artist", "record producer", "songwriter", "drill"]):
+                    return item.get("id")
+            if data:
+                return data[0].get("id")
+    except Exception:
+        pass
+    return ""
+
+
+def get_identifiers_for_qid(qid: str) -> Dict[str, str]:
+    """
+    Extracts P1902 (Spotify ID) and P2003 (Instagram username) directly 
+    from the Wikidata item JSON claim tree.
+    """
+    if not qid:
+        return {"spotify_id": "N/A", "instagram_handle": "N/A"}
+
+    url = f"https://www.wikidata.org/wiki/Special:EntityData/{qid}.json"
+    try:
+        res = requests.get(url, headers=HEADERS, timeout=10)
+        if res.status_code == 200:
+            entities = res.json().get("entities", {})
+            entity = entities.get(qid, {})
+            claims = entity.get("claims", {})
+
+            # P1902: Spotify artist ID
+            spotify_id = "N/A"
+            if "P1902" in claims:
+                for statement in claims["P1902"]:
+                    mainsnak = statement.get("mainsnak", {})
+                    datavalue = mainsnak.get("datavalue", {})
+                    val = datavalue.get("value")
+                    if val and isinstance(val, str):
+                        spotify_id = val
+                        break
+
+            # P2003: Instagram username
+            instagram_handle = "N/A"
+            if "P2003" in claims:
+                for statement in claims["P2003"]:
+                    mainsnak = statement.get("mainsnak", {})
+                    datavalue = mainsnak.get("datavalue", {})
+                    val = datavalue.get("value")
+                    if val and isinstance(val, str):
+                        instagram_handle = val
+                        break
+
+            return {
+                "spotify_id": spotify_id,
+                "instagram_handle": instagram_handle
+            }
+    except Exception:
+        pass
+
+    return {"spotify_id": "N/A", "instagram_handle": "N/A"}
+
+
+def resolve_pilot_cohort(artist_names: List[str]) -> pd.DataFrame:
+    rows = []
+    for name in artist_names:
+        qid = search_wikidata_qid(name)
+        if qid:
+            ids = get_identifiers_for_qid(qid)
+            rows.append({
+                "name": name,
+                "wikidata_qid": qid,
+                "wikidata_uri": f"https://www.wikidata.org/wiki/{qid}",
+                "spotify_id": ids["spotify_id"],
+                "instagram_handle": ids["instagram_handle"],
+                "status": "RESOLVED"
+            })
+        else:
+            rows.append({
+                "name": name,
+                "wikidata_qid": "N/A",
+                "wikidata_uri": "N/A",
+                "spotify_id": "N/A",
+                "instagram_handle": "N/A",
+                "status": "NOT_FOUND"
+            })
+    return pd.DataFrame(rows)