feat: integrate daily velocity ingestion, circuit breaker, clean requirements, and resilient batch loader
This commit is contained in:
Srilekha committed
1 parent
66db687a58
commit
90b4956635
6 files changed
+832
-948
No files matched your search
@@ -1,18 +1,12 @@
|
||||
"""
|
||||
main.py
|
||||
Unified Command-Line Interface (CLI) Entry Point for HIFFI Artist Intelligence Pipeline.
|
||||
|
||||
Supported Commands:
|
||||
python main.py run # Run the batch ingestion and signal merge pipeline
|
||||
python main.py audit # Run automated anomaly detection and data health checks
|
||||
python main.py report # Print the executive provenance and ranking report
|
||||
python main.py test # Execute the automated Pytest governance suite
|
||||
python main.py all # Run complete end-to-end pipeline (audit -> test -> report)
|
||||
HIFFI Artist Intelligence Pipeline CLI Orchestrator.
|
||||
Supports unified commands: run, audit, test, report, all.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
@@ -20,92 +14,82 @@ if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from core.health_detector import PipelineHealthDetector
|
||||
from pipeline.batch_500_runner import BatchMultiPlatformRunner
|
||||
from pipeline.batch_500_runner import main as run_batch_pipeline
|
||||
|
||||
|
||||
def cmd_run(args):
|
||||
"""Executes the batch ingestion and multi-platform scoring pipeline."""
|
||||
print("\n[HIFFI CLI] Initializing Multi-Platform Batch Runner...")
|
||||
runner = BatchMultiPlatformRunner("enriched_artists.csv")
|
||||
runner.run_full_pipeline(batch_size=args.batch_size, delay_range=(0.5, 1.0))
|
||||
def run_pipeline():
|
||||
print("\n[HIFFI CLI] Executing Batch Multi-Platform Signal Ingestion & Scoring...")
|
||||
run_batch_pipeline()
|
||||
|
||||
|
||||
def cmd_audit(args):
|
||||
"""Runs automated health checks and anomaly detection."""
|
||||
print("\n[HIFFI CLI] Executing Automated Pipeline Health Detector...")
|
||||
def run_audit():
|
||||
print("\n[HIFFI CLI] Executing Automated Pipeline Health & Anomaly Detector...")
|
||||
detector = PipelineHealthDetector()
|
||||
detector.print_diagnostic_report()
|
||||
anomalies_df, summary = detector.run_health_audit()
|
||||
|
||||
print("=" * 85)
|
||||
print("HIFFI PIPELINE AUTOMATED HEALTH & ANOMALY DETECTOR")
|
||||
print("=" * 85)
|
||||
print(f"Total Cohort Rows Evaluated: {summary['total_rows']}")
|
||||
print(f"Total Anomalies Flagged: {summary['total_anomalies']}")
|
||||
print(f"Actionable Unresolved (P0/P1): {summary['actionable_p0_p1']}")
|
||||
print(f"Data Governance Breaches: {summary['governance_breaches']}")
|
||||
print("-" * 85)
|
||||
if not anomalies_df.empty:
|
||||
print(anomalies_df.to_string(index=False))
|
||||
else:
|
||||
print("No anomalous signals detected across active cohort.")
|
||||
print("=" * 85)
|
||||
|
||||
|
||||
def cmd_report(args):
|
||||
"""Prints executive ranking and data provenance summaries."""
|
||||
def run_tests():
|
||||
print("\n[HIFFI CLI] Running QA Governance Test Suite via Pytest...")
|
||||
test_file = PROJECT_ROOT / "test_multiplatform_ranking.py"
|
||||
cmd = [sys.executable, "-m", "pytest", str(test_file), "-v"]
|
||||
result = subprocess.run(cmd)
|
||||
if result.returncode != 0:
|
||||
print("\n[HIFFI CLI] QA Governance Test Suite FAILED.")
|
||||
sys.exit(result.returncode)
|
||||
|
||||
|
||||
def run_report():
|
||||
print("\n[HIFFI CLI] Generating Executive Provenance Report...")
|
||||
report_script = PROJECT_ROOT / "pipeline" / "generate_report.py"
|
||||
subprocess.run([sys.executable, str(report_script)], check=True)
|
||||
if report_script.exists():
|
||||
subprocess.run([sys.executable, str(report_script)])
|
||||
else:
|
||||
print(f"[HIFFI CLI] Report script not found at {report_script}")
|
||||
|
||||
|
||||
def cmd_test(args):
|
||||
"""Executes the pytest QA test suite."""
|
||||
print("\n[HIFFI CLI] Running QA Governance Test Suite...")
|
||||
test_file = PROJECT_ROOT / "test_multiplatform_ranking.py"
|
||||
cmd = [sys.executable, "-m", "pytest", "-v" if args.verbose else "-q", str(test_file)]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
def cmd_all(args):
|
||||
"""Executes end-to-end workflow: Audit -> Pytest -> Report."""
|
||||
print("\n" + "=" * 80)
|
||||
def run_all():
|
||||
print("=" * 80)
|
||||
print("HIFFI PIPELINE: EXECUTING FULL END-TO-END VALIDATION & REPORTING")
|
||||
print("=" * 80)
|
||||
|
||||
# 1. Health Audit
|
||||
cmd_audit(args)
|
||||
|
||||
# 2. Pytest Governance Check
|
||||
cmd_test(args)
|
||||
|
||||
# 3. Final Report
|
||||
cmd_report(args)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="hiffi",
|
||||
description="HIFFI Multi-Platform Artist Intelligence Data Pipeline CLI",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True, help="Pipeline action to execute")
|
||||
|
||||
# Command: run
|
||||
p_run = subparsers.add_parser("run", help="Run batch ingestion & multi-platform scoring")
|
||||
p_run.add_argument("--batch-size", type=int, default=15, help="Batch size for checkpoint saves")
|
||||
p_run.set_defaults(func=cmd_run)
|
||||
|
||||
# Command: audit
|
||||
p_audit = subparsers.add_parser("audit", help="Run automated anomaly detection & health checks")
|
||||
p_audit.set_defaults(func=cmd_audit)
|
||||
|
||||
# Command: report
|
||||
p_report = subparsers.add_parser("report", help="Display executive movers & provenance report")
|
||||
p_report.set_defaults(func=cmd_report)
|
||||
|
||||
# Command: test
|
||||
p_test = subparsers.add_parser("test", help="Run the automated QA test suite")
|
||||
p_test.add_argument("-v", "--verbose", action="store_true", help="Verbose pytest output")
|
||||
p_test.set_defaults(func=cmd_test)
|
||||
|
||||
# Command: all
|
||||
p_all = subparsers.add_parser("all", help="Execute complete health check, QA suite, and executive report")
|
||||
p_all.add_argument("-v", "--verbose", action="store_true", help="Verbose pytest output")
|
||||
p_all.set_defaults(func=cmd_all)
|
||||
|
||||
return parser
|
||||
run_audit()
|
||||
run_tests()
|
||||
run_report()
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
parser = argparse.ArgumentParser(description="HIFFI Artist Intelligence Multi-Platform Pipeline CLI")
|
||||
parser.add_argument(
|
||||
"action",
|
||||
choices=["run", "audit", "test", "report", "all"],
|
||||
help="Pipeline command to execute: 'run' (ingest & score), 'audit' (health checks), 'test' (pytest suite), 'report' (executive movers), or 'all' (audit + test + report)."
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
if args.action == "run":
|
||||
run_pipeline()
|
||||
elif args.action == "audit":
|
||||
run_audit()
|
||||
elif args.action == "test":
|
||||
run_tests()
|
||||
elif args.action == "report":
|
||||
run_report()
|
||||
elif args.action == "all":
|
||||
run_all()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user