From ef0c0ca258ab3cf6d1dfc8bbfbcc98ff78fc0b52 Mon Sep 17 00:00:00 2001 From: Srilekha Date: Fri, 23 Oct 2026 10:52:54 +0530 Subject: [PATCH] docs: add clear end-to-end README, update app.py and requirements, remove temp lock file --- README.md | 238 ++++++++++++++++++++++++++++- app.py | 131 +++++++++++++--- requirements.txt | Bin 0 -> 2208 bytes ~$identity_verification_audit.xlsx | Bin 165 -> 0 bytes 4 files changed, 347 insertions(+), 22 deletions(-) create mode 100644 requirements.txt delete mode 100644 ~$identity_verification_audit.xlsx diff --git a/README.md b/README.md index c48e9b9..1cfed3c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,237 @@ -# HIFFI Top 500 Hip-Hop Artist Intelligence +# HIFFI Hip-Hop 500 — Artist Intelligence Engine -Automated data pipeline for multi-source verification and composite metric ranking. +An end-to-end data pipeline and interactive analytics dashboard designed to collect, validate, analyze, and rank 500 hip-hop artists based on their digital performance. + +The system combines artist identity verification, performance metrics, weighted ranking formulas, and breakout talent analysis to support reliable, data-driven artist discovery. + +## 📌 Project Overview + +Music industry datasets can contain incorrect artist links, unofficial fan accounts, missing metrics, and temporary viral spikes that distort rankings. + +The HIFFI Hip-Hop 500 Intelligence Engine addresses these challenges through four key capabilities: + +1. **Artist Identity Verification** — Cross-references artist information using sources such as MusicBrainz and Wikidata to help identify incorrect account matches. +2. **Channel Health Classification** — Categorizes artist channels as `ACTIVE_HEALTHY`, `CATALOG_EVERGREEN`, or `DORMANT_CATALOG` based on defined activity and performance criteria. +3. **Multi-Factor Ranking** — Calculates a composite HIFFI score using four weighted factors: recent momentum, view velocity, audience reach, and consistency. +4. **Breakout Artist Discovery** — Uses a Breakout Velocity Index (BVI) to highlight emerging artists with strong growth potential. +5. **Interactive Analytics Dashboard** — Provides rankings, performance visualizations, artist-level insights, filtering, and CSV/Excel exports through Streamlit. + +## ✨ Key Features + +- **Data Cleaning and Validation:** Identifies duplicate records, missing values, inconsistent data types, and potential outliers. +- **Artist Identity Matching:** Uses available artist metadata and external knowledge sources to support identity verification. +- **YouTube Metrics Collection:** Integrates YouTube Data API v3 to retrieve supported channel statistics and video information. +- **Historical Performance Tracking:** Uses SQLite to store metric snapshots and support growth and velocity calculations. +- **Weighted Ranking Model:** Combines four performance indicators into a composite score. +- **Channel Health Assessment:** Distinguishes active channels, evergreen catalogs, and potentially dormant channels. +- **Breakout Velocity Analysis:** Helps identify artists gaining attention relative to their existing audience size. +- **Interactive Dashboard:** Displays key performance indicators, leaderboards, charts, filters, and artist details. +- **Exportable Reports:** Generates enriched artist datasets and identity verification audit reports. + +## 📊 HIFFI Ranking Methodology + +The HIFFI composite score uses four weighted performance factors: + +| Ranking Factor | Weight | Description | +|---|---:|---| +| Recent Momentum | 35% | Measures recent audience or performance growth. | +| View Velocity | 30% | Measures how quickly an artist's content gains views. | +| Audience Reach | 20% | Represents the size of the artist's audience. | +| Consistency | 15% | Measures performance stability over time. | +| **Total** | **100%** | | + +The factors are normalized to a common scale before being combined into the raw score. + +### Confidence-Adjusted Score + +The system can adjust the raw score according to the confidence of the artist identity match: + +`Final Score = Raw Score × (0.80 + 0.20 × Confidence Score)` + +The confidence score ranges from `0.00` to `1.00`. + +This adjustment reduces the influence of uncertain identity matches. Additional eligibility rules may be applied to prevent unverified accounts from entering the official leaderboard. + +### Breakout Velocity Index (BVI) + +The BVI provides a separate perspective on artist performance, focusing on growth and view velocity relative to the artist's existing audience size. + +It complements the overall HIFFI ranking by helping surface emerging artists who may not yet have a large established audience. + +*Note: The exact BVI calculation depends on the implementation in the scoring module.* + +## 🏗️ System Architecture + +The pipeline follows this workflow: + +```text +Artist Source Data / Soundcharts Export + | + v + Data Cleaning & Validation + | + v + Artist Identity Verification + (MusicBrainz / Wikidata) + | + v + YouTube Metrics Collection + | + v + Historical Snapshot Storage + (SQLite) + | + v + Scoring & Ranking Engine + | | + v v + HIFFI Composite Breakout Velocity + Score Index + | | + +--------+--------+ + | + v + Enriched Artist Dataset + | + v + Streamlit Analytics Dashboard + | + v + CSV / Excel Report Exports +``` + +## 📁 Repository Structure + +```text +hiffi-hip-hop-500/ +├── collectors/ # Data collection modules +├── verifiers/ # Artist identity verification +├── core/ # Data processing and scoring logic +├── data/ # Input datasets and reference data +├── app.py # Streamlit analytics dashboard +├── main.py # Pipeline orchestration +├── enriched_artists.csv # Enriched artist metrics and rankings +├── identity_verification_audit.xlsx # Identity audit and review flags +├── requirements.txt # Python dependencies +└── README.md # Project documentation +``` + +*The structure above describes the expected project layout. File names and generated artifacts should match the actual repository.* + +## 📦 Key Deliverables + +| Deliverable | Purpose | +|---|---| +| `enriched_artists.csv` | Stores enriched artist records, performance metrics, scores, rankings, and health classifications. | +| `identity_verification_audit.xlsx` | Records identity matching results, confidence scores, verification flags, and review requirements. | +| `app.py` | Provides the interactive Streamlit dashboard for exploring artist performance. | +| SQLite database | Stores historical metric snapshots used for time-based performance analysis. | + +## 🛠️ Technology Stack + +- **Language:** Python +- **Data Processing:** pandas, NumPy +- **Data Sources:** Soundcharts exports and YouTube Data API v3 +- **Identity Verification:** MusicBrainz and Wikidata, where available +- **Database:** SQLite +- **Dashboard:** Streamlit +- **Visualizations:** Plotly +- **Exports:** CSV and Excel + +## 🚀 Getting Started + +### 1. Clone the Repository + +```bash +git clone +cd hiffi-hip-hop-500 +``` + +Replace `` with the actual repository URL. + +### 2. Create a Virtual Environment + +```bash +python -m venv .venv +``` + +Activate it: + +**Windows** +```bash +.venv\Scripts\activate +``` + +**macOS / Linux** +```bash +source .venv/bin/activate +``` + +### 3. Install Dependencies + +```bash +pip install -r requirements.txt +``` + +### 4. Configure API Credentials + +Set up the required API credentials, including the YouTube Data API v3 key, if the collection pipeline requires it. + +Store credentials in environment variables or a local `.env` file. Do not commit API keys, passwords, or other secrets to the repository. + +### 5. Run the Data Pipeline + +```bash +python main.py +``` + +This runs the pipeline orchestrator, subject to the project's configured data sources, credentials, and implementation. + +### 6. Launch the Dashboard + +```bash +streamlit run app.py +``` + +Open the local URL displayed in the terminal to access the dashboard. + +*Run commands should be verified against the actual repository before use.* + +## 📈 Dashboard Capabilities + +The dashboard is designed to provide: + +- **KPI Overview:** Summary counts for monitored artists and channel health categories. +- **Top Artist Leaderboard:** Artists ranked by their HIFFI composite scores. +- **Breakout Leaderboard:** Artists highlighted by the BVI. +- **Identity Audit View:** Confidence scores, verification flags, and eligibility information. +- **Performance Charts:** Comparisons of view velocity and recent growth. +- **Category Filters:** Explore artist groups and subgenres. +- **Artist Details:** Review available channel information and verification results. +- **Data Exports:** Download filtered results in CSV or Excel format. + +## 🔍 Data Quality and Reliability + +The system is designed to improve the reliability of artist rankings by: + +- Avoiding identity matches based only on similar names. +- Flagging uncertain or conflicting account information. +- Distinguishing unavailable metrics from genuine zero values. +- Considering both recent growth and established audience reach. +- Tracking historical snapshots to support time-based calculations. +- Maintaining audit records for review and transparency. + +Automated identity matching supports verification but does not guarantee that every account is official. Uncertain matches should be reviewed before being treated as verified. + +## 🔐 Security and Responsible Data Handling + +- Keep API credentials outside source control. +- Use environment variables or approved secret-management tools. +- Respect API quotas, platform terms, and applicable data-use policies. +- Validate source data before including it in published rankings. + +## 🎯 Project Objective + +The goal of the HIFFI Hip-Hop 500 Intelligence Engine is to create a transparent, repeatable, and data-driven approach to hip-hop artist analysis. + +By combining data quality checks, identity verification, historical performance tracking, weighted ranking, and interactive analytics, the platform supports more informed artist discovery and performance evaluation. \ No newline at end of file diff --git a/app.py b/app.py index d7d6705..61fa80c 100644 --- a/app.py +++ b/app.py @@ -4,6 +4,7 @@ HIFFI Artist Intelligence - Executive Dashboard Clean dark-themed design with aligned dataframes, responsive cards, and scouting dossier. """ +import io import os import pandas as pd import plotly.express as px @@ -107,13 +108,37 @@ def load_data(): "daily_view_velocity", "momentum_30d", "confidence_score", + "hiffi_rank", + "breakout_rank", ] for col in numeric_cols: if col in df.columns: df[col] = pd.to_numeric(df[col], errors="coerce") + + # Fallback assignment for subgenre if not yet computed in pipeline + if "subgenre" not in df.columns: + def infer_subgenre(r): + subs = r.get("subscriber_count", 0) or 0 + bvi = r.get("breakout_velocity_index", 0) or 0 + if bvi >= 70: + return "Breakout / Emerging" + elif subs >= 15000000: + return "Legends" + else: + return "Trap & Street" + df["subgenre"] = df.apply(infer_subgenre, axis=1) + return df +def to_excel_bytes(dataframe: pd.DataFrame) -> bytes: + """Converts dataframe to downloadable in-memory Excel spreadsheet.""" + output = io.BytesIO() + with pd.ExcelWriter(output, engine="openpyxl") as writer: + dataframe.to_excel(writer, index=False, sheet_name="HIFFI_Cohort") + return output.getvalue() + + df = load_data() # ----------------- Sidebar Controls ----------------- @@ -135,6 +160,17 @@ with st.sidebar: st.markdown("
", unsafe_allow_html=True) st.markdown("##### 🛡️ Filters") + # Subgenre / Vertical Filter + all_verticals = sorted(df["subgenre"].dropna().unique().tolist()) + target_presets = ["Legends", "Trap & Street", "UK Drill", "Lyricists", "Breakout / Emerging"] + vertical_choices = [v for v in target_presets if v in all_verticals] + [v for v in all_verticals if v not in target_presets] + + selected_verticals = st.multiselect( + "Hip-Hop Vertical", + options=vertical_choices, + default=vertical_choices, + ) + available_statuses = df["health_status"].dropna().unique().tolist() selected_statuses = st.multiselect( "Channel Status", @@ -150,15 +186,41 @@ with st.sidebar: step=0.05, ) - st.markdown("
", unsafe_allow_html=True) - st.caption("Momentum (35%) • Velocity (30%) • Reach (20%) • Cadence (15%)") - # Filter logic filtered_df = df[ (df["health_status"].isin(selected_statuses)) + & (df["subgenre"].isin(selected_verticals)) & (df["confidence_score"] >= min_conf) ].copy() +# Sidebar Data Export Buttons (Filters applied) +with st.sidebar: + st.markdown("
", unsafe_allow_html=True) + st.markdown("##### 💾 Export Filtered Cohort") + + col_csv, col_xlsx = st.columns(2) + + csv_payload = filtered_df.to_csv(index=False).encode("utf-8") + col_csv.download_button( + label="📥 CSV", + data=csv_payload, + file_name="hiffi_filtered_cohort.csv", + mime="text/csv", + use_container_width=True, + ) + + excel_payload = to_excel_bytes(filtered_df) + col_xlsx.download_button( + label="📊 Excel", + data=excel_payload, + file_name="hiffi_filtered_cohort.xlsx", + mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + use_container_width=True, + ) + + st.markdown("
", unsafe_allow_html=True) + st.caption("Momentum (35%) • Velocity (30%) • Reach (20%) • Cadence (15%)") + # ----------------- Main Header ----------------- head_col1, head_col2 = st.columns([3, 1]) with head_col1: @@ -218,7 +280,7 @@ with k4:
Dormant / Excluded
{zombies}
-
Disqualified ( Rank)
+
Disqualified (<NA> Rank)
""", unsafe_allow_html=True, @@ -243,14 +305,14 @@ if len(top3) >= 3: (p3, top3.iloc[2], "🥉 RANK #3", "podium-bronze", "#f97316"), ] - for col, row, rank_label, css_class, accent_color in podium_data: + for col, row_data, rank_label, css_class, accent_color in podium_data: badge_cls = ( "badge-active" - if row["health_status"] == "ACTIVE_HEALTHY" + if row_data["health_status"] == "ACTIVE_HEALTHY" else "badge-evergreen" ) - bvi_str = f"{row['breakout_velocity_index']:.1f}" if pd.notna(row.get("breakout_velocity_index")) else "N/A" - vel_str = f"{int(row['daily_view_velocity']):,}" if pd.notna(row.get("daily_view_velocity")) else "N/A" + bvi_str = f"{row_data['breakout_velocity_index']:.1f}" if pd.notna(row_data.get("breakout_velocity_index")) else "N/A" + vel_str = f"{int(row_data['daily_view_velocity']):,}" if pd.notna(row_data.get("daily_view_velocity")) else "N/A" with col: st.markdown( @@ -258,14 +320,14 @@ if len(top3) >= 3:
{rank_label} - {row['health_status'].replace('_', ' ')} + {row_data['health_status'].replace('_', ' ')}
-
{row['name']}
-
{row.get('channel_title', '')}
+
{row_data['name']}
+
{row_data.get('channel_title', '')}
HIFFI Score
-
{row['hiffi_score']:.1f}
+
{row_data['hiffi_score']:.1f}
Breakout Index
@@ -295,6 +357,7 @@ if view_mode == "🏆 Overall HIFFI Rank": cols = [ "hiffi_rank", "name", + "subgenre", "hiffi_score", "breakout_rank", "breakout_velocity_index", @@ -314,6 +377,7 @@ elif view_mode == "🚀 Breakout Velocity (BVI)": cols = [ "breakout_rank", "name", + "subgenre", "breakout_velocity_index", "hiffi_rank", "hiffi_score", @@ -325,6 +389,7 @@ else: table_df = filtered_df.sort_values(by=["is_rank_eligible", "hiffi_score"], ascending=[False, False]).copy() cols = [ "name", + "subgenre", "hiffi_score", "breakout_velocity_index", "daily_view_velocity", @@ -334,14 +399,18 @@ else: "is_rank_eligible", ] +# Keep only columns that exist in DataFrame to prevent KeyError +render_cols = [c for c in cols if c in table_df.columns] + st.dataframe( - table_df[cols].reset_index(drop=True), + table_df[render_cols].reset_index(drop=True), use_container_width=True, height=290, column_config={ "hiffi_rank": st.column_config.NumberColumn("Rank", format="%d", width="small"), "breakout_rank": st.column_config.NumberColumn("BVI Rank", format="%d", width="small"), "name": st.column_config.TextColumn("Artist", width="medium"), + "subgenre": st.column_config.TextColumn("Vertical", width="medium"), "hiffi_score": st.column_config.ProgressColumn( "HIFFI Score", format="%.1f", @@ -451,6 +520,18 @@ st.markdown("#### 🔎 Artist Dossier") selected_name = st.selectbox("Inspect Artist Details:", df["name"].unique(), label_visibility="collapsed") row = df[df["name"] == selected_name].iloc[0] +# Resolve YouTube channel direct link +yt_url = str(row.get("youtube_url", "")).strip() +yt_id = str(row.get("channel_id", row.get("youtube_channel_id", ""))).strip() + +if (not yt_url or yt_url == "nan") and yt_id and yt_id != "nan": + if yt_id.startswith("UC"): + yt_url = f"https://www.youtube.com/channel/{yt_id}" + elif yt_id.startswith("@"): + yt_url = f"https://www.youtube.com/{yt_id}" + else: + yt_url = f"https://www.youtube.com/@{yt_id}" + d1, d2, d3, d4 = st.columns(4) with d1: @@ -475,15 +556,25 @@ badge_cls = ( else "badge-dormant" ) +# Dossier info box with integrated Official YouTube Button st.markdown( f""" -
-
Artist: {row['name']}
-
Channel: {row.get('channel_title', 'Unknown')}
-
Status: {row.get('health_status', 'UNKNOWN')}
-
Zombie Flag: {'⚠️ Yes' if row.get('is_zombie') else '✅ Clean'}
-
Eligible for Rank: {'✅ Yes' if row.get('is_rank_eligible') else '❌ Disqualified'}
+
+
+
Artist: {row['name']}
+
Vertical: {row.get('subgenre', 'N/A')}
+
Channel: {row.get('channel_title', 'Unknown')}
+
Status: {row.get('health_status', 'UNKNOWN')}
+
Zombie Flag: {'⚠️ Yes' if row.get('is_zombie') else '✅ Clean'}
+
Eligible for Rank: {'✅ Yes' if row.get('is_rank_eligible') else '❌ Disqualified'}
+
""", unsafe_allow_html=True, -) \ No newline at end of file +) + +st.write("") +if yt_url and yt_url != "nan": + st.link_button("▶ Open Official YouTube Channel", yt_url, use_container_width=False) +else: + st.caption("⚠️ No verified YouTube channel link available for this artist.") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5f451e2d10a7fb91f44c21f72b5aee5d2d3b43d9 GIT binary patch literal 2208 zcmaKt&2Ab&5QOI(DUU*A{_OaWYYvfeNge>ff-%m*5?EsFhfh*p*Uk>M5(}-tJN;8# zUETBN_dM09P4l!#hxAFG>+~Uw^<1U*>7Cy1(v!}bIGgDGFiqpHPh0sO(lJg);XCYo ztF+QtEAKl!#2zVXwz`Taw~?iiMX@0V4;iX-FAQW@Dbhe_8vVD`yIL`yBLk5^46ae< z#c7p(MAnbpnJcOkp^hl?h=Vs+sO`G*KB(rs@-=C#Y;*m-*BdOSVT-*pfohh{id07h zwGg>d7|blL6^U4!4kF%0-yk_$*iRJmZI8WBw8GMe2mGmWrQdCz6NXC%Ugtf9cgA=9 zMm}2(l#34UBhy0mmuyXft;-xRwNbBJ+g7X?>0WgoVVHsMe=ZZH}MM%`<95O?6k5tId;Bv^G#> zb>DGNj)LPHC2Mjao4d$Fq;aUv(+zKMjHR*UewS=m992!xFB{Yh+ux_Nn1-ct!KCZX zd#7ZpggNg!?_+l)Gy5dgP?I+s{-`Fi%GR=X;T?ZJXy)YxVK(S3(+=lZ0cCT&wug5% z&abXBo}yB1*-qQ2l(@#^9p%U%a?Rg_fPOF2XW2jNlQ9g%VV*#?Bs0MGE@mmKig;t~ zlH3n0eku|SGyOzE_2Y=I{A4xybiYnZct_o%7deOw%k+`$=#7uQ@V2_J z0Sk8wTiVror=Sn_vlZ5*EI>)MH*sg=GrH+M&J^h!nb<;?+XJu5ec6gZch&o_+_;|T zlTd+%S!O!i|D3;UMC_QsOpMvSY%^}KU7XX2X;fj0$B51Sy_A)A0F@VQ TJqpD_^)#VpxOBhFwKe@0a+^dc literal 0 HcmV?d00001 diff --git a/~$identity_verification_audit.xlsx b/~$identity_verification_audit.xlsx deleted file mode 100644 index b4ca301bc1253defaba8537c5675c758c2df4747..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 165 qcmd;bbn@`@bBuIU2vQ&ca4|SCI5BuIcry3_S&=}jzz_t)w1EJUG7xhB