hiffi-artist-intelligence/pipeline/export_review_queue.py

"""
pipeline/export_review_queue.py
Extracts uncertain Instagram accounts and suspicious candidates into an actionable review queue.
Provides programmatic validation and import workflows for human auditors.
"""

import argparse
import json
import sys
from datetime import datetime, timezone
from pathlib import Path

import pandas as pd

PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

DATASET_PATH = PROJECT_ROOT / "data" / "hiffi_multiplatform_500.csv"
REVIEW_QUEUE_PATH = PROJECT_ROOT / "data" / "review_queue.csv"
OVERRIDES_PATH = PROJECT_ROOT / "data" / "handle_overrides.json"


def export_queue(include_unmatched: bool = False):
    """Exports candidates needing human auditing into a structured CSV."""
    if not DATASET_PATH.exists():
        print(f"[Review Queue] Dataset not found at: {DATASET_PATH}")
        return

    df = pd.read_csv(DATASET_PATH)

    # Filter targets: UNCERTAIN_NEEDS_REVIEW or high-rank artists missing verification
    criteria = df["instagram_verification_status"] == "UNCERTAIN_NEEDS_REVIEW"
    if include_unmatched:
        criteria = criteria | (df["instagram_verification_status"] == "UNMATCHED_NO_HANDLE")

    # Always pull Top 100 artists that lack verification
    top_100_unverified = (df["production_hiffi_rank"] <= 100) & (
        ~df["instagram_verification_status"].isin(["MANUAL_VERIFIED", "AUTOMATED_VERIFIED"])
    )
    queue_df = df[criteria | top_100_unverified].copy()

    if queue_df.empty:
        print("[Review Queue] No accounts currently require manual review.")
        return

    # Sort prioritized by production impact (top YouTube baseline ranks first)
    queue_df = queue_df.sort_values(by="production_hiffi_rank", ascending=True)

    # Structure audit fields
    review_columns = [
        "name",
        "production_hiffi_rank",
        "instagram_handle",
        "instagram_followers",
        "instagram_confidence_score",
        "instagram_verification_status",
        "instagram_audit_evidence",
        "channel_title"
    ]
    present_cols = [c for c in review_columns if c in queue_df.columns]
    export_df = queue_df[present_cols].copy()

    # Add auditor decision columns
    export_df["auditor_action"] = "REVIEW_PENDING"  # Options: ACCEPT, REJECT, MODIFY
    export_df["corrected_handle"] = ""
    export_df["auditor_notes"] = ""

    REVIEW_QUEUE_PATH.parent.mkdir(parents=True, exist_ok=True)
    export_df.to_csv(REVIEW_QUEUE_PATH, index=False)

    print("=" * 80)
    print("HIFFI INSTAGRAM CANDIDATE REVIEW QUEUE")
    print("=" * 80)
    print(f"Total Rows Flagged for Review: {len(export_df)}")
    print(f"Top 100 Artists In Queue:      {(export_df['production_hiffi_rank'] <= 100).sum()}")
    print(f"Queue Exported To:             {REVIEW_QUEUE_PATH}")
    print("\nAuditor Workflow:")
    print(" 1. Open 'data/review_queue.csv' in Excel or VS Code.")
    print(" 2. Set 'auditor_action' to 'ACCEPT', 'REJECT', or 'MODIFY'.")
    print(" 3. If 'MODIFY', provide the verified handle in 'corrected_handle'.")
    print(" 4. Run: python pipeline/export_review_queue.py --apply")
    print("=" * 80)


def apply_decisions():
    """Reads review_queue.csv and imports accepted/modified decisions into handle_overrides.json."""
    if not REVIEW_QUEUE_PATH.exists():
        print(f"[Review Queue] No active review queue found at {REVIEW_QUEUE_PATH}")
        return

    review_df = pd.read_csv(REVIEW_QUEUE_PATH)
    overrides = {}
    if OVERRIDES_PATH.exists():
        try:
            with open(OVERRIDES_PATH, "r", encoding="utf-8") as f:
                overrides = json.load(f)
        except Exception:
            overrides = {}

    applied_count = 0
    rejected_count = 0

    for _, row in review_df.iterrows():
        action = str(row.get("auditor_action", "")).strip().upper()
        artist_name = str(row.get("name", "")).strip()

        if action == "ACCEPT":
            handle = str(row.get("instagram_handle", "")).replace("@", "").strip()
            followers = row.get("instagram_followers")
            if handle and handle.lower() not in ("nan", "none", ""):
                overrides[artist_name] = {
                    "instagram_handle": handle,
                    "instagram_followers": int(followers) if pd.notna(followers) else None,
                    "curated_at": datetime.now(timezone.utc).isoformat(),
                    "auditor_notes": str(row.get("auditor_notes", "Auditor approved candidate"))
                }
                applied_count += 1

        elif action == "MODIFY":
            handle = str(row.get("corrected_handle", "")).replace("@", "").strip()
            followers = row.get("instagram_followers")
            if handle and handle.lower() not in ("nan", "none", ""):
                overrides[artist_name] = {
                    "instagram_handle": handle,
                    "instagram_followers": int(followers) if pd.notna(followers) else None,
                    "curated_at": datetime.now(timezone.utc).isoformat(),
                    "auditor_notes": str(row.get("auditor_notes", "Auditor corrected handle"))
                }
                applied_count += 1

        elif action == "REJECT":
            rejected_count += 1

    with open(OVERRIDES_PATH, "w", encoding="utf-8") as f:
        json.dump(overrides, f, indent=2)

    print(f"[Review Queue] Applied {applied_count} manual approvals/modifications to {OVERRIDES_PATH.name}.")
    print(f"[Review Queue] Recorded {rejected_count} explicitly rejected candidates.")
    print("Run 'python main.py all' to recompile the audited dataset with new overrides.")


def main():
    parser = argparse.ArgumentParser(description="HIFFI Instagram Candidate Review Queue Exporter")
    parser.add_argument("--apply", action="store_true", help="Apply auditor decisions back to handle_overrides.json")
    parser.add_argument("--include-unmatched", action="store_true", help="Include records with missing handles")
    args = parser.parse_args()

    if args.apply:
        apply_decisions()
    else:
        export_queue(include_unmatched=args.include_unmatched)


if __name__ == "__main__":
    main()