feat(pipeline): complete end-to-end multiplatform ingestion, QA governance, zero anomalies, and docs

pipeline/export_review_queue.py

@@ -0,0 +1,156 @@
+"""
+pipeline/export_review_queue.py
+Extracts uncertain Instagram accounts and suspicious candidates into an actionable review queue.
+Provides programmatic validation and import workflows for human auditors.
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pandas as pd
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+DATASET_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"
+REVIEW_QUEUE_PATH = PROJECT_ROOT / "data" / "review_queue.csv"
+OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json"
+
+
+def export_queue(include_unmatched: bool = False):
+    """Exports candidates needing human auditing into a structured CSV."""
+    if not DATASET_PATH.exists():
+        print(f"[Review Queue] Dataset not found at: {DATASET_PATH}")
+        return
+
+    df = pd.read_csv(DATASET_PATH)
+
+    # Filter targets: UNCERTAIN_NEEDS_REVIEW or high-rank artists missing verification
+    criteria = df["instagram_verification_status"] == "UNCERTAIN_NEEDS_REVIEW"
+    if include_unmatched:
+        criteria = criteria | (df["instagram_verification_status"] == "UNMATCHED_NO_HANDLE")
+
+    # Always pull Top 100 artists that lack verification
+    top_100_unverified = (df["production_hiffi_rank"] <= 100) & (
+        ~df["instagram_verification_status"].isin(["MANUAL_VERIFIED", "AUTOMATED_VERIFIED"])
+    )
+    queue_df = df[criteria | top_100_unverified].copy()
+
+    if queue_df.empty:
+        print("[Review Queue] No accounts currently require manual review.")
+        return
+
+    # Sort prioritized by production impact (top YouTube baseline ranks first)
+    queue_df = queue_df.sort_values(by="production_hiffi_rank", ascending=True)
+
+    # Structure audit fields
+    review_columns = [
+        "name",
+        "production_hiffi_rank",
+        "instagram_handle",
+        "instagram_followers",
+        "instagram_confidence_score",
+        "instagram_verification_status",
+        "instagram_audit_evidence",
+        "channel_title"
+    ]
+    present_cols = [c for c in review_columns if c in queue_df.columns]
+    export_df = queue_df[present_cols].copy()
+
+    # Add auditor decision columns
+    export_df["auditor_action"] = "REVIEW_PENDING"  # Options: ACCEPT, REJECT, MODIFY
+    export_df["corrected_handle"] = ""
+    export_df["auditor_notes"] = ""
+
+    REVIEW_QUEUE_PATH.parent.mkdir(parents=True, exist_ok=True)
+    export_df.to_csv(REVIEW_QUEUE_PATH, index=False)
+
+    print("=" * 80)
+    print("HIFFI INSTAGRAM CANDIDATE REVIEW QUEUE")
+    print("=" * 80)
+    print(f"Total Rows Flagged for Review: {len(export_df)}")
+    print(f"Top 100 Artists In Queue:      {(export_df['production_hiffi_rank'] <= 100).sum()}")
+    print(f"Queue Exported To:             {REVIEW_QUEUE_PATH}")
+    print("\nAuditor Workflow:")
+    print(" 1. Open 'data/review_queue.csv' in Excel or VS Code.")
+    print(" 2. Set 'auditor_action' to 'ACCEPT', 'REJECT', or 'MODIFY'.")
+    print(" 3. If 'MODIFY', provide the verified handle in 'corrected_handle'.")
+    print(" 4. Run: python pipeline/export_review_queue.py --apply")
+    print("=" * 80)
+
+
+def apply_decisions():
+    """Reads review_queue.csv and imports accepted/modified decisions into handle_overrides.json."""
+    if not REVIEW_QUEUE_PATH.exists():
+        print(f"[Review Queue] No active review queue found at {REVIEW_QUEUE_PATH}")
+        return
+
+    review_df = pd.read_csv(REVIEW_QUEUE_PATH)
+    overrides = {}
+    if OVERRIDES_PATH.exists():
+        try:
+            with open(OVERRIDES_PATH, "r", encoding="utf-8") as f:
+                overrides = json.load(f)
+        except Exception:
+            overrides = {}
+
+    applied_count = 0
+    rejected_count = 0
+
+    for _, row in review_df.iterrows():
+        action = str(row.get("auditor_action", "")).strip().upper()
+        artist_name = str(row.get("name", "")).strip()
+
+        if action == "ACCEPT":
+            handle = str(row.get("instagram_handle", "")).replace("@", "").strip()
+            followers = row.get("instagram_followers")
+            if handle and handle.lower() not in ("nan", "none", ""):
+                overrides[artist_name] = {
+                    "instagram_handle": handle,
+                    "instagram_followers": int(followers) if pd.notna(followers) else None,
+                    "curated_at": datetime.now(timezone.utc).isoformat(),
+                    "auditor_notes": str(row.get("auditor_notes", "Auditor approved candidate"))
+                }
+                applied_count += 1
+
+        elif action == "MODIFY":
+            handle = str(row.get("corrected_handle", "")).replace("@", "").strip()
+            followers = row.get("instagram_followers")
+            if handle and handle.lower() not in ("nan", "none", ""):
+                overrides[artist_name] = {
+                    "instagram_handle": handle,
+                    "instagram_followers": int(followers) if pd.notna(followers) else None,
+                    "curated_at": datetime.now(timezone.utc).isoformat(),
+                    "auditor_notes": str(row.get("auditor_notes", "Auditor corrected handle"))
+                }
+                applied_count += 1
+
+        elif action == "REJECT":
+            rejected_count += 1
+
+    with open(OVERRIDES_PATH, "w", encoding="utf-8") as f:
+        json.dump(overrides, f, indent=2)
+
+    print(f"[Review Queue] Applied {applied_count} manual approvals/modifications to {OVERRIDES_PATH.name}.")
+    print(f"[Review Queue] Recorded {rejected_count} explicitly rejected candidates.")
+    print("Run 'python main.py all' to recompile the audited dataset with new overrides.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="HIFFI Instagram Candidate Review Queue Exporter")
+    parser.add_argument("--apply", action="store_true", help="Apply auditor decisions back to handle_overrides.json")
+    parser.add_argument("--include-unmatched", action="store_true", help="Include records with missing handles")
+    args = parser.parse_args()
+
+    if args.apply:
+        apply_decisions()
+    else:
+        export_queue(include_unmatched=args.include_unmatched)
+
+
+if __name__ == "__main__":
+    main()