feat: complete multiplatform pipeline hardening, neutral anchor scoring, and automated health checks
This commit is contained in:
Srilekha committed
1 parent
56cbd08248
commit
66db687a58
11 files changed
+1097
-1005
No files matched your search
@@ -1,196 +1,112 @@
|
||||
"""
|
||||
main.py
|
||||
Production CSV-driven batch pipeline:
|
||||
- Loads seed artists from data/seed_artists_500.csv via RosterLoader
|
||||
- Checkpoint-aware batch execution (saves every CHUNK_SIZE artists)
|
||||
- tqdm progress tracking with ETA
|
||||
- Exports:
|
||||
1. enriched_artists.csv (Full master dataset)
|
||||
2. identity_verification_audit.xlsx (Separate multi-tab Excel workbook)
|
||||
Unified Command-Line Interface (CLI) Entry Point for HIFFI Artist Intelligence Pipeline.
|
||||
|
||||
Supported Commands:
|
||||
python main.py run # Run the batch ingestion and signal merge pipeline
|
||||
python main.py audit # Run automated anomaly detection and data health checks
|
||||
python main.py report # Print the executive provenance and ranking report
|
||||
python main.py test # Execute the automated Pytest governance suite
|
||||
python main.py all # Run complete end-to-end pipeline (audit -> test -> report)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import pandas as pd
|
||||
from dotenv import load_dotenv
|
||||
from tqdm import tqdm
|
||||
import argparse
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from collectors.youtube_collector import YouTubeCollector
|
||||
from verifiers.musicbrainz_verifier import MusicBrainzVerifier
|
||||
from verifiers.wikidata_verifier import WikidataVerifier
|
||||
from core.confidence_scorer import ConfidenceScorer
|
||||
from core.ranking_engine import HIFFIRankingEngine
|
||||
from core.health_detector import ChannelHealthDetector
|
||||
from data.roster_loader import RosterLoader
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
load_dotenv()
|
||||
|
||||
CSV_ROSTER_PATH = "data/seed_artists_500.csv"
|
||||
OUTPUT_CSV_PATH = "enriched_artists.csv"
|
||||
OUTPUT_EXCEL_PATH = "identity_verification_audit.xlsx"
|
||||
CHUNK_SIZE = 25
|
||||
from core.health_detector import PipelineHealthDetector
|
||||
from pipeline.batch_500_runner import BatchMultiPlatformRunner
|
||||
|
||||
|
||||
def export_verification_excel(df: pd.DataFrame, excel_path: str = OUTPUT_EXCEL_PATH):
|
||||
"""
|
||||
Exports a clean, multi-tab Excel workbook dedicated to identity verification audits.
|
||||
"""
|
||||
# Columns relevant for identity verification
|
||||
audit_cols = [
|
||||
"name",
|
||||
"channel_title",
|
||||
"identifier",
|
||||
"confidence_score",
|
||||
"confidence_tier",
|
||||
"status_flag",
|
||||
"mb_verified",
|
||||
"wd_verified",
|
||||
"cross_referenced",
|
||||
]
|
||||
|
||||
# Filter available columns defensively
|
||||
existing_cols = [c for c in audit_cols if c in df.columns]
|
||||
audit_df = df[existing_cols].copy()
|
||||
|
||||
# Sort by confidence score descending
|
||||
if "confidence_score" in audit_df.columns:
|
||||
audit_df = audit_df.sort_values(by="confidence_score", ascending=False)
|
||||
|
||||
# Flagged subset for low confidence or failed lookups
|
||||
flagged_df = audit_df[
|
||||
(audit_df["confidence_score"] < 0.50)
|
||||
| (audit_df["status_flag"].isin(["FAILED_LOOKUP", "LOOKUP_ERROR"]))
|
||||
].copy()
|
||||
|
||||
with pd.ExcelWriter(excel_path, engine="openpyxl") as writer:
|
||||
audit_df.to_excel(writer, sheet_name="Identity Verification Audit", index=False)
|
||||
flagged_df.to_excel(writer, sheet_name="Flagged & Unresolved", index=False)
|
||||
|
||||
print(f"📊 Separate Identity Verification workbook exported to '{excel_path}'")
|
||||
def cmd_run(args):
|
||||
"""Executes the batch ingestion and multi-platform scoring pipeline."""
|
||||
print("\n[HIFFI CLI] Initializing Multi-Platform Batch Runner...")
|
||||
runner = BatchMultiPlatformRunner("enriched_artists.csv")
|
||||
runner.run_full_pipeline(batch_size=args.batch_size, delay_range=(0.5, 1.0))
|
||||
|
||||
|
||||
def run_batch_pipeline(roster_path: str = CSV_ROSTER_PATH, force_refresh: bool = False):
|
||||
print(f"\n========================================================")
|
||||
print(f" HIFFI PRODUCTION INGESTION PIPELINE")
|
||||
print(f" Source: {roster_path} | Checkpoint Chunk: {CHUNK_SIZE}")
|
||||
print(f"========================================================\n")
|
||||
def cmd_audit(args):
|
||||
"""Runs automated health checks and anomaly detection."""
|
||||
print("\n[HIFFI CLI] Executing Automated Pipeline Health Detector...")
|
||||
detector = PipelineHealthDetector()
|
||||
detector.print_diagnostic_report()
|
||||
|
||||
loader = RosterLoader(file_path=roster_path)
|
||||
artists = loader.load_roster()
|
||||
total_artists = len(artists)
|
||||
|
||||
collector = YouTubeCollector()
|
||||
mb_verifier = MusicBrainzVerifier()
|
||||
wd_verifier = WikidataVerifier()
|
||||
conf_scorer = ConfidenceScorer()
|
||||
ranking_engine = HIFFIRankingEngine()
|
||||
health_detector = ChannelHealthDetector()
|
||||
def cmd_report(args):
|
||||
"""Prints executive ranking and data provenance summaries."""
|
||||
print("\n[HIFFI CLI] Generating Executive Provenance Report...")
|
||||
report_script = PROJECT_ROOT / "pipeline" / "generate_report.py"
|
||||
subprocess.run([sys.executable, str(report_script)], check=True)
|
||||
|
||||
all_records = []
|
||||
|
||||
progress_bar = tqdm(
|
||||
total=total_artists,
|
||||
desc="Processing Roster",
|
||||
unit="artist",
|
||||
ncols=95,
|
||||
bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]"
|
||||
def cmd_test(args):
|
||||
"""Executes the pytest QA test suite."""
|
||||
print("\n[HIFFI CLI] Running QA Governance Test Suite...")
|
||||
test_file = PROJECT_ROOT / "test_multiplatform_ranking.py"
|
||||
cmd = [sys.executable, "-m", "pytest", "-v" if args.verbose else "-q", str(test_file)]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
def cmd_all(args):
|
||||
"""Executes end-to-end workflow: Audit -> Pytest -> Report."""
|
||||
print("\n" + "=" * 80)
|
||||
print("HIFFI PIPELINE: EXECUTING FULL END-TO-END VALIDATION & REPORTING")
|
||||
print("=" * 80)
|
||||
|
||||
# 1. Health Audit
|
||||
cmd_audit(args)
|
||||
|
||||
# 2. Pytest Governance Check
|
||||
cmd_test(args)
|
||||
|
||||
# 3. Final Report
|
||||
cmd_report(args)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="hiffi",
|
||||
description="HIFFI Multi-Platform Artist Intelligence Data Pipeline CLI",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True, help="Pipeline action to execute")
|
||||
|
||||
for i, artist in enumerate(artists):
|
||||
name = artist["name"]
|
||||
ident = artist["identifier"]
|
||||
progress_bar.set_postfix_str(f"Analyzing: {name[:18]}")
|
||||
# Command: run
|
||||
p_run = subparsers.add_parser("run", help="Run batch ingestion & multi-platform scoring")
|
||||
p_run.add_argument("--batch-size", type=int, default=15, help="Batch size for checkpoint saves")
|
||||
p_run.set_defaults(func=cmd_run)
|
||||
|
||||
record = {**artist}
|
||||
# Command: audit
|
||||
p_audit = subparsers.add_parser("audit", help="Run automated anomaly detection & health checks")
|
||||
p_audit.set_defaults(func=cmd_audit)
|
||||
|
||||
try:
|
||||
# 1. Fetch channel metrics
|
||||
yt = collector.get_channel_metrics(
|
||||
ident,
|
||||
fetch_recent_videos=True,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
record.update(yt)
|
||||
# Command: report
|
||||
p_report = subparsers.add_parser("report", help="Display executive movers & provenance report")
|
||||
p_report.set_defaults(func=cmd_report)
|
||||
|
||||
# 2. Verify identity across MusicBrainz & Wikidata
|
||||
mb = mb_verifier.verify_artist(name)
|
||||
wd = wd_verifier.verify_artist(name)
|
||||
# Command: test
|
||||
p_test = subparsers.add_parser("test", help="Run the automated QA test suite")
|
||||
p_test.add_argument("-v", "--verbose", action="store_true", help="Verbose pytest output")
|
||||
p_test.set_defaults(func=cmd_test)
|
||||
|
||||
yt_with_name = {**yt, "name": name}
|
||||
conf_result = conf_scorer.calculate_score(
|
||||
yt_metrics=yt_with_name, mb_result=mb, wd_result=wd
|
||||
)
|
||||
# Command: all
|
||||
p_all = subparsers.add_parser("all", help="Execute complete health check, QA suite, and executive report")
|
||||
p_all.add_argument("-v", "--verbose", action="store_true", help="Verbose pytest output")
|
||||
p_all.set_defaults(func=cmd_all)
|
||||
|
||||
# Record verification sub-metrics for Excel export
|
||||
record["confidence_score"] = float(conf_result.get("composite_confidence", 0.0))
|
||||
record["confidence_tier"] = conf_result.get("confidence_tier", "Unknown")
|
||||
sub_scores = conf_result.get("sub_scores", {})
|
||||
record["mb_verified"] = bool(sub_scores.get("musicbrainz", 0.0) == 1.0)
|
||||
record["wd_verified"] = bool(sub_scores.get("wikidata", 0.0) == 1.0)
|
||||
record["cross_referenced"] = bool(sub_scores.get("cross_referenced", False))
|
||||
return parser
|
||||
|
||||
except Exception as e:
|
||||
record.update({
|
||||
"channel_id": None,
|
||||
"channel_title": None,
|
||||
"subscriber_count": None,
|
||||
"lifetime_views": None,
|
||||
"video_count": None,
|
||||
"recent_avg_views": None,
|
||||
"daily_view_velocity": None,
|
||||
"momentum_30d": None,
|
||||
"status_flag": "LOOKUP_ERROR",
|
||||
"data_complete": 0,
|
||||
"confidence_score": 0.0,
|
||||
"confidence_tier": "Error / Unresolved",
|
||||
"mb_verified": False,
|
||||
"wd_verified": False,
|
||||
"cross_referenced": False,
|
||||
"error_notes": str(e),
|
||||
})
|
||||
|
||||
all_records.append(record)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Intermediate checkpoint every CHUNK_SIZE
|
||||
if (i + 1) % CHUNK_SIZE == 0 or (i + 1) == total_artists:
|
||||
temp_df = pd.DataFrame(all_records)
|
||||
ranked_temp = ranking_engine.score_and_rank(temp_df)
|
||||
health_temp = health_detector.evaluate_dataframe(ranked_temp)
|
||||
health_temp.to_csv(OUTPUT_CSV_PATH, index=False)
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Final scoring and health detection
|
||||
print("\n--- Finalizing Official HIFFI Ranks & Breakout Indices ---")
|
||||
df_raw = pd.DataFrame(all_records)
|
||||
ranked_df = ranking_engine.score_and_rank(df_raw)
|
||||
final_df = health_detector.evaluate_dataframe(ranked_df)
|
||||
|
||||
# 1. Save Master CSV
|
||||
final_df.to_csv(OUTPUT_CSV_PATH, index=False)
|
||||
print(f"✅ Master dataset written to '{OUTPUT_CSV_PATH}'")
|
||||
|
||||
# 2. Save Separate Identity Verification Excel
|
||||
export_verification_excel(final_df, excel_path=OUTPUT_EXCEL_PATH)
|
||||
|
||||
# Summary Leaderboard
|
||||
print("\n--- Top 10 HIFFI Market Leaders ---")
|
||||
hiffi_top10 = final_df[final_df["is_rank_eligible"] == True].sort_values(
|
||||
by="hiffi_rank", ascending=True
|
||||
).head(10)
|
||||
summary_cols = [
|
||||
"hiffi_rank",
|
||||
"name",
|
||||
"hiffi_score",
|
||||
"breakout_rank",
|
||||
"breakout_velocity_index",
|
||||
"daily_view_velocity",
|
||||
"health_status",
|
||||
]
|
||||
print(hiffi_top10[summary_cols].to_string(index=False))
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_batch_pipeline(roster_path=CSV_ROSTER_PATH, force_refresh=False)
|
||||
main()
|
||||
Reference in new issue
Block a user