feat(pipeline): complete end-to-end multiplatform ingestion, QA governance, zero anomalies, and docs

This commit is contained in:
Srilekha committed 2026-10-24 15:57:20 +05:30
1 parent 62aea7a85b
commit 05da5050df
13 files changed
+1874 -1103

No files matched your search

+35 -31
View File
@@ -1,11 +1,10 @@
"""
main.py
HIFFI Artist Intelligence Pipeline CLI Orchestrator.
Supports unified commands: run, audit, test, report, all.
Unified Command-Line Interface (CLI) for the HIFFI Multi-Platform Intelligence Pipeline.
Orchestrates batch ingestion, provenance audit, anomaly detection, QA tests, review queue, and reporting.
"""
import argparse
import subprocess
import sys
from pathlib import Path
@@ -13,8 +12,10 @@ PROJECT_ROOT = Path(__file__).resolve().parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from core.health_detector import PipelineHealthDetector
import pytest
from core.health_detector import main as run_health_audit
from pipeline.batch_500_runner import main as run_batch_pipeline
from reports.provenance_summary import main as run_provenance_summary
def run_pipeline():
@@ -24,41 +25,30 @@ def run_pipeline():
def run_audit():
print("\n[HIFFI CLI] Executing Automated Pipeline Health & Anomaly Detector...")
detector = PipelineHealthDetector()
anomalies_df, summary = detector.run_health_audit()
print("=" * 85)
print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
print("=" * 85)
print(f"Total Cohort Rows Evaluated: {summary['total_rows']}")
print(f"Total Anomalies Flagged: {summary['total_anomalies']}")
print(f"Actionable Unresolved (P0/P1): {summary['actionable_p0_p1']}")
print(f"Data Governance Breaches: {summary['governance_breaches']}")
print("-" * 85)
if not anomalies_df.empty:
print(anomalies_df.to_string(index=False))
else:
print("No anomalous signals detected across active cohort.")
print("=" * 85)
run_health_audit()
def run_tests():
print("\n[HIFFI CLI] Running QA Governance Test Suite via Pytest...")
test_file = PROJECT_ROOT / "test_multiplatform_ranking.py"
cmd = [sys.executable, "-m", "pytest", str(test_file), "-v"]
result = subprocess.run(cmd)
if result.returncode != 0:
retcode = pytest.main(["-v", str(test_file)])
if retcode != 0:
print("\n[HIFFI CLI] QA Governance Test Suite FAILED.")
sys.exit(result.returncode)
sys.exit(retcode)
def run_report():
print("\n[HIFFI CLI] Generating Executive Provenance Report...")
report_script = PROJECT_ROOT / "pipeline" / "generate_report.py"
if report_script.exists():
subprocess.run([sys.executable, str(report_script)])
run_provenance_summary()
def run_review(apply_mode: bool = False, include_unmatched: bool = False):
print("\n[HIFFI CLI] Managing Instagram Candidate Review Queue...")
from pipeline.export_review_queue import apply_decisions, export_queue
if apply_mode:
apply_decisions()
else:
print(f"[HIFFI CLI] Report script not found at {report_script}")
export_queue(include_unmatched=include_unmatched)
def run_all():
@@ -72,11 +62,23 @@ def run_all():
def main():
parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Multi-Platform Pipeline CLI")
parser = argparse.ArgumentParser(
description="HIFFI 500-Artist Multi-Platform Pipeline CLI Engine"
)
parser.add_argument(
"action",
choices=["run", "audit", "test", "report", "all"],
help="Pipeline command to execute: 'run' (ingest & score), 'audit' (health checks), 'test' (pytest suite), 'report' (executive movers), or 'all' (run + audit + test + report)."
choices=["run", "audit", "test", "report", "review", "all"],
help="Pipeline command to execute: 'run', 'audit', 'test', 'report', 'review', or 'all'."
)
parser.add_argument(
"--apply",
action="store_true",
help="Apply review queue decisions to handle_overrides.json (used with 'review')"
)
parser.add_argument(
"--include-unmatched",
action="store_true",
help="Include records with missing candidate handles in review export"
)
args = parser.parse_args()
@@ -89,6 +91,8 @@ def main():
run_tests()
elif args.action == "report":
run_report()
elif args.action == "review":
run_review(apply_mode=args.apply, include_unmatched=args.include_unmatched)
elif args.action == "all":
run_all()