feat(pipeline): complete end-to-end multiplatform ingestion, QA governance, zero anomalies, and docs
This commit is contained in:
1 parent
62aea7a85b
commit
05da5050df
13 files changed
+1874
-1103
No files matched your search
@@ -1,11 +1,10 @@
|
||||
"""
|
||||
main.py
|
||||
HIFFI Artist Intelligence Pipeline CLI Orchestrator.
|
||||
Supports unified commands: run, audit, test, report, all.
|
||||
Unified Command-Line Interface (CLI) for the HIFFI Multi-Platform Intelligence Pipeline.
|
||||
Orchestrates batch ingestion, provenance audit, anomaly detection, QA tests, review queue, and reporting.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -13,8 +12,10 @@ PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from core.health_detector import PipelineHealthDetector
|
||||
import pytest
|
||||
from core.health_detector import main as run_health_audit
|
||||
from pipeline.batch_500_runner import main as run_batch_pipeline
|
||||
from reports.provenance_summary import main as run_provenance_summary
|
||||
|
||||
|
||||
def run_pipeline():
|
||||
@@ -24,41 +25,30 @@ def run_pipeline():
|
||||
|
||||
def run_audit():
|
||||
print("\n[HIFFI CLI] Executing Automated Pipeline Health & Anomaly Detector...")
|
||||
detector = PipelineHealthDetector()
|
||||
anomalies_df, summary = detector.run_health_audit()
|
||||
|
||||
print("=" * 85)
|
||||
print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
|
||||
print("=" * 85)
|
||||
print(f"Total Cohort Rows Evaluated: {summary['total_rows']}")
|
||||
print(f"Total Anomalies Flagged: {summary['total_anomalies']}")
|
||||
print(f"Actionable Unresolved (P0/P1): {summary['actionable_p0_p1']}")
|
||||
print(f"Data Governance Breaches: {summary['governance_breaches']}")
|
||||
print("-" * 85)
|
||||
if not anomalies_df.empty:
|
||||
print(anomalies_df.to_string(index=False))
|
||||
else:
|
||||
print("No anomalous signals detected across active cohort.")
|
||||
print("=" * 85)
|
||||
run_health_audit()
|
||||
|
||||
|
||||
def run_tests():
|
||||
print("\n[HIFFI CLI] Running QA Governance Test Suite via Pytest...")
|
||||
test_file = PROJECT_ROOT / "test_multiplatform_ranking.py"
|
||||
cmd = [sys.executable, "-m", "pytest", str(test_file), "-v"]
|
||||
result = subprocess.run(cmd)
|
||||
if result.returncode != 0:
|
||||
retcode = pytest.main(["-v", str(test_file)])
|
||||
if retcode != 0:
|
||||
print("\n[HIFFI CLI] QA Governance Test Suite FAILED.")
|
||||
sys.exit(result.returncode)
|
||||
sys.exit(retcode)
|
||||
|
||||
|
||||
def run_report():
|
||||
print("\n[HIFFI CLI] Generating Executive Provenance Report...")
|
||||
report_script = PROJECT_ROOT / "pipeline" / "generate_report.py"
|
||||
if report_script.exists():
|
||||
subprocess.run([sys.executable, str(report_script)])
|
||||
run_provenance_summary()
|
||||
|
||||
|
||||
def run_review(apply_mode: bool = False, include_unmatched: bool = False):
|
||||
print("\n[HIFFI CLI] Managing Instagram Candidate Review Queue...")
|
||||
from pipeline.export_review_queue import apply_decisions, export_queue
|
||||
if apply_mode:
|
||||
apply_decisions()
|
||||
else:
|
||||
print(f"[HIFFI CLI] Report script not found at {report_script}")
|
||||
export_queue(include_unmatched=include_unmatched)
|
||||
|
||||
|
||||
def run_all():
|
||||
@@ -72,11 +62,23 @@ def run_all():
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Multi-Platform Pipeline CLI")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="HIFFI 500-Artist Multi-Platform Pipeline CLI Engine"
|
||||
)
|
||||
parser.add_argument(
|
||||
"action",
|
||||
choices=["run", "audit", "test", "report", "all"],
|
||||
help="Pipeline command to execute: 'run' (ingest & score), 'audit' (health checks), 'test' (pytest suite), 'report' (executive movers), or 'all' (run + audit + test + report)."
|
||||
choices=["run", "audit", "test", "report", "review", "all"],
|
||||
help="Pipeline command to execute: 'run', 'audit', 'test', 'report', 'review', or 'all'."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="Apply review queue decisions to handle_overrides.json (used with 'review')"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-unmatched",
|
||||
action="store_true",
|
||||
help="Include records with missing candidate handles in review export"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -89,6 +91,8 @@ def main():
|
||||
run_tests()
|
||||
elif args.action == "report":
|
||||
run_report()
|
||||
elif args.action == "review":
|
||||
run_review(apply_mode=args.apply, include_unmatched=args.include_unmatched)
|
||||
elif args.action == "all":
|
||||
run_all()
|
||||
|
||||
|
||||
Reference in new issue
Block a user