""" pipeline/export_review_queue.py Extracts uncertain Instagram accounts and suspicious candidates into an actionable review queue. Provides programmatic validation and import workflows for human auditors. """ import argparse import json import sys from datetime import datetime, timezone from pathlib import Path import pandas as pd PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) DATASET_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv" REVIEW_QUEUE_PATH = PROJECT_ROOT / "data" / "review_queue.csv" OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json" def export_queue(include_unmatched: bool = False): """Exports candidates needing human auditing into a structured CSV.""" if not DATASET_PATH.exists(): print(f"[Review Queue] Dataset not found at: {DATASET_PATH}") return df = pd.read_csv(DATASET_PATH) # Filter targets: UNCERTAIN_NEEDS_REVIEW or high-rank artists missing verification criteria = df["instagram_verification_status"] == "UNCERTAIN_NEEDS_REVIEW" if include_unmatched: criteria = criteria | (df["instagram_verification_status"] == "UNMATCHED_NO_HANDLE") # Always pull Top 100 artists that lack verification top_100_unverified = (df["production_hiffi_rank"] <= 100) & ( ~df["instagram_verification_status"].isin(["MANUAL_VERIFIED", "AUTOMATED_VERIFIED"]) ) queue_df = df[criteria | top_100_unverified].copy() if queue_df.empty: print("[Review Queue] No accounts currently require manual review.") return # Sort prioritized by production impact (top YouTube baseline ranks first) queue_df = queue_df.sort_values(by="production_hiffi_rank", ascending=True) # Structure audit fields review_columns = [ "name", "production_hiffi_rank", "instagram_handle", "instagram_followers", "instagram_confidence_score", "instagram_verification_status", "instagram_audit_evidence", "channel_title" ] present_cols = [c for c in review_columns if c in queue_df.columns] export_df = queue_df[present_cols].copy() # Add auditor decision columns export_df["auditor_action"] = "REVIEW_PENDING" # Options: ACCEPT, REJECT, MODIFY export_df["corrected_handle"] = "" export_df["auditor_notes"] = "" REVIEW_QUEUE_PATH.parent.mkdir(parents=True, exist_ok=True) export_df.to_csv(REVIEW_QUEUE_PATH, index=False) print("=" * 80) print("HIFFI INSTAGRAM CANDIDATE REVIEW QUEUE") print("=" * 80) print(f"Total Rows Flagged for Review: {len(export_df)}") print(f"Top 100 Artists In Queue: {(export_df['production_hiffi_rank'] <= 100).sum()}") print(f"Queue Exported To: {REVIEW_QUEUE_PATH}") print("\nAuditor Workflow:") print(" 1. Open 'data/review_queue.csv' in Excel or VS Code.") print(" 2. Set 'auditor_action' to 'ACCEPT', 'REJECT', or 'MODIFY'.") print(" 3. If 'MODIFY', provide the verified handle in 'corrected_handle'.") print(" 4. Run: python pipeline/export_review_queue.py --apply") print("=" * 80) def apply_decisions(): """Reads review_queue.csv and imports accepted/modified decisions into handle_overrides.json.""" if not REVIEW_QUEUE_PATH.exists(): print(f"[Review Queue] No active review queue found at {REVIEW_QUEUE_PATH}") return review_df = pd.read_csv(REVIEW_QUEUE_PATH) overrides = {} if OVERRIDES_PATH.exists(): try: with open(OVERRIDES_PATH, "r", encoding="utf-8") as f: overrides = json.load(f) except Exception: overrides = {} applied_count = 0 rejected_count = 0 for _, row in review_df.iterrows(): action = str(row.get("auditor_action", "")).strip().upper() artist_name = str(row.get("name", "")).strip() if action == "ACCEPT": handle = str(row.get("instagram_handle", "")).replace("@", "").strip() followers = row.get("instagram_followers") if handle and handle.lower() not in ("nan", "none", ""): overrides[artist_name] = { "instagram_handle": handle, "instagram_followers": int(followers) if pd.notna(followers) else None, "curated_at": datetime.now(timezone.utc).isoformat(), "auditor_notes": str(row.get("auditor_notes", "Auditor approved candidate")) } applied_count += 1 elif action == "MODIFY": handle = str(row.get("corrected_handle", "")).replace("@", "").strip() followers = row.get("instagram_followers") if handle and handle.lower() not in ("nan", "none", ""): overrides[artist_name] = { "instagram_handle": handle, "instagram_followers": int(followers) if pd.notna(followers) else None, "curated_at": datetime.now(timezone.utc).isoformat(), "auditor_notes": str(row.get("auditor_notes", "Auditor corrected handle")) } applied_count += 1 elif action == "REJECT": rejected_count += 1 with open(OVERRIDES_PATH, "w", encoding="utf-8") as f: json.dump(overrides, f, indent=2) print(f"[Review Queue] Applied {applied_count} manual approvals/modifications to {OVERRIDES_PATH.name}.") print(f"[Review Queue] Recorded {rejected_count} explicitly rejected candidates.") print("Run 'python main.py all' to recompile the audited dataset with new overrides.") def main(): parser = argparse.ArgumentParser(description="HIFFI Instagram Candidate Review Queue Exporter") parser.add_argument("--apply", action="store_true", help="Apply auditor decisions back to handle_overrides.json") parser.add_argument("--include-unmatched", action="store_true", help="Include records with missing handles") args = parser.parse_args() if args.apply: apply_decisions() else: export_queue(include_unmatched=args.include_unmatched) if __name__ == "__main__": main()