validate_model.py

"""
FairValue Model Validation Suite
=================================
Tests the model against 30 known real-world elite transfers to verify:
  1. Prediction Direction  — higher-quality players get higher values
  2. Magnitude Sanity     — predictions are in the right ballpark (not £2m for Mbappe)
  3. Monotonicity         — age/contract/injury sliders move predictions the right way
  4. Ranking Accuracy     — model rank-orders players correctly (Spearman correlation)
  5. Risk Flag Logic      — harder contracts/older players get discounts

Run from project root:
    python validate_model.py
"""

import os
import sys
import numpy as np
import pandas as pd
import xgboost as xgb
from scipy.stats import spearmanr
from src.features.build_features import calculate_risk_score

# ── Config ────────────────────────────────────────────────────────────────────
MODEL_PATH = "fairvalue_xgboost.json"
DATA_PATH  = "data/processed/app_features.csv"
EUR_TO_GBP = 0.85
TOLERANCE_PCT = 60   # We expect predictions within ±60% of real fee (generous for elite)

PASS = "✅ PASS"
FAIL = "❌ FAIL"
WARN = "⚠️  WARN"

# ── Known Ground Truth (real transfer fees in £m) ─────────────────────────────
GROUND_TRUTH = [
    # name,                     real_fee_gbp_m, age, contract_yrs, injury_days, mkt_val_eur_m
    ("Neymar Jr",                     222, 25, 5, 12,  180),
    ("Kylian Mbappe",                 168, 19, 6,  5,  150),
    ("Philippe Coutinho",             136, 25, 5, 10,  120),
    ("Joao Felix",                    126, 19, 6, 25,  110),
    ("Antoine Griezmann",             120, 27, 5,  0,  115),
    ("Harry Maguire",                  83, 25, 6, 20,   90),
    ("Romelu Lukaku",                  91, 28, 5, 30,  100),
    ("Jack Grealish",                  93, 25, 6, 15,  100),
    ("Jadon Sancho",                   80, 21, 5,  8,  110),
    ("Declan Rice",                   105, 24, 5,  5,  110),
    ("Moises Caicedo",                115, 22, 6, 10,  105),
    ("Enzo Fernandez",                107, 22, 6,  5,  120),
    ("Jude Bellingham",               103, 19, 5,  5,  150),
    ("Erling Haaland",                 57, 21, 6, 20,  145),  # low fee due to release clause
    ("Darwin Nunez",                   89, 22, 5, 18,   90),
    ("Ruben Neves",                    54, 26, 5, 15,   55),
    ("Andre Onana",                    50, 27, 5, 10,   50),
]

def load_model_and_data():
    if not os.path.exists(MODEL_PATH):
        print(f"{FAIL}  Model file not found: {MODEL_PATH}")
        sys.exit(1)
    if not os.path.exists(DATA_PATH):
        print(f"{FAIL}  Data file not found: {DATA_PATH}")
        sys.exit(1)

    model = xgb.XGBRegressor()
    model.load_model(MODEL_PATH)

    df = pd.read_csv(DATA_PATH)
    mv_map = {c: 'market_value_in_eur' for c in df.columns
              if 'market' in c.lower() and 'value' in c.lower()}
    if mv_map:
        df.rename(columns=mv_map, inplace=True)
    df = df.loc[:, ~df.columns.duplicated()].copy()

    size = os.path.getsize(MODEL_PATH)
    print(f"  Model loaded: {MODEL_PATH} ({size:,} bytes)")
    print(f"  Data loaded:  {DATA_PATH} ({len(df):,} rows, {len(df.columns)} features)")
    return model, df


def predict(model, df, age, contract_yrs, injury_days, mkt_val_eur_m):
    """Build a feature row and return predicted transfer fee in £m."""
    expected_cols = model.feature_names_in_
    base = df.median(numeric_only=True).to_frame().T
    base['Age'] = age
    base['Contract_Years_Left'] = contract_yrs
    if 'Injury_Days_Total_24m' in base.columns:
        base['Injury_Days_Total_24m'] = injury_days
    if 'market_value_in_eur' in base.columns:
        base['market_value_in_eur'] = mkt_val_eur_m * 1_000_000
    # Recompute derived risk flags to match the test inputs
    base = calculate_risk_score(base,
                                contract_col='Contract_Years_Left',
                                age_col='Age',
                                injury_col='Injury_Days_Total_24m')
    X = base.reindex(columns=expected_cols, fill_value=0)
    log_pred = model.predict(X)[0]
    predicted_eur = max(np.expm1(log_pred), 0.0)
    return predicted_eur * EUR_TO_GBP / 1_000_000


def separator(title=""):
    w = 70
    if title:
        print(f"\n{'─' * 4} {title} {'─' * (w - len(title) - 6)}")
    else:
        print("─" * w)


# ─────────────────────────────────────────────────────────────────────────────
def test_1_model_health(model, df):
    separator("TEST 1 — Model Health Check")
    size = os.path.getsize(MODEL_PATH)
    feats = len(model.feature_names_in_)

    checks = [
        ("Model file > 10 KB (not corrupted)",  size > 10_000),
        ("Model has > 5 features",              feats > 5),
        ("Data has > 1,000 rows",               len(df) > 1_000),
        ("'Age' in feature set",                'Age' in model.feature_names_in_),
        ("'Contract_Years_Left' in features",   'Contract_Years_Left' in model.feature_names_in_),
    ]
    all_pass = True
    for label, ok in checks:
        status = PASS if ok else FAIL
        print(f"  {status}  {label}")
        if not ok:
            all_pass = False
    return all_pass


# ─────────────────────────────────────────────────────────────────────────────
def test_2_magnitude_sanity(model, df):
    separator("TEST 2 — Magnitude Sanity (known elite transfers)")
    print(f"  {'Player':<25} {'Real £m':>8} {'Pred £m':>8} {'Error%':>8}  {'Status'}")
    print(f"  {'─'*25} {'─'*8} {'─'*8} {'─'*8}  {'─'*6}")

    errors = []
    results = []
    for name, real_fee, age, contract, injuries, mkt_val in GROUND_TRUTH:
        pred = predict(model, df, age, contract, injuries, mkt_val)
        err_pct = abs(pred - real_fee) / real_fee * 100
        errors.append(err_pct)
        ok = err_pct <= TOLERANCE_PCT
        status = PASS if ok else WARN
        print(f"  {name:<25} {real_fee:>7.0f}  {pred:>7.1f}  {err_pct:>7.1f}%  {status}")
        results.append((name, real_fee, pred))

    median_err = np.median(errors)
    within_60  = sum(e <= 60  for e in errors)
    within_100 = sum(e <= 100 for e in errors)
    print(f"\n  Median error:        {median_err:.1f}%")
    print(f"  Within ±60%:         {within_60}/{len(errors)} players")
    print(f"  Within ±100%:        {within_100}/{len(errors)} players")

    grade = PASS if within_60 >= len(errors) * 0.6 else WARN
    print(f"\n  Verdict: {grade}  (target ≥60% of players within ±60%)")
    return results


# ─────────────────────────────────────────────────────────────────────────────
def test_3_monotonicity(model, df):
    separator("TEST 3 — Monotonicity (sliders move in the right direction)")
    # Market value held CONSTANT at £80m so we isolate age/contract/injury effects.
    # Without fixing MV, the df.median() row has different implicit values per age.
    base_age, base_contract, base_injuries, base_mkt = 25, 3.0, 15, 80

    base_pred = predict(model, df, base_age, base_contract, base_injuries, base_mkt)

    checks = [
        ("Younger player (19) valued MORE than 32yo  [MV fixed at £80m]",
         predict(model, df, 19, base_contract, base_injuries, base_mkt),
         predict(model, df, 32, base_contract, base_injuries, base_mkt),
         True),

        ("Longer contract (5yr) valued MORE than 0.5yr  [MV fixed at £80m]",
         predict(model, df, base_age, 5.0, base_injuries, base_mkt),
         predict(model, df, base_age, 0.5, base_injuries, base_mkt),
         True),

        ("Healthy player (0 days) valued MORE than 150 days  [MV fixed at £80m]",
         predict(model, df, base_age, base_contract, 0,   base_mkt),
         predict(model, df, base_age, base_contract, 150, base_mkt),
         True),

        ("Higher MV (£120m) valued MORE than lower MV (£30m)",
         predict(model, df, base_age, base_contract, base_injuries, 120),
         predict(model, df, base_age, base_contract, base_injuries, 30),
         True),
    ]

    all_pass = True
    for label, higher_val, lower_val, should_be_higher in checks:
        ok = (higher_val > lower_val) == should_be_higher
        status = PASS if ok else FAIL
        direction = "correct" if ok else "INVERTED!"
        print(f"  {status}  {label}")
        print(f"           → Higher case: \u00a3{higher_val:.1f}m | Lower case: \u00a3{lower_val:.1f}m  [{direction}]")
        if not ok:
            all_pass = False

    return all_pass


# ─────────────────────────────────────────────────────────────────────────────
def test_4_ranking_accuracy(results):
    separator("TEST 4 — Ranking Accuracy (Spearman Correlation)")
    real_fees  = [r[1] for r in results]
    pred_fees  = [r[2] for r in results]
    corr, pval = spearmanr(real_fees, pred_fees)

    print(f"  Spearman ρ (rank correlation): {corr:.3f}")
    print(f"  p-value:                       {pval:.4f}")
    print(f"  Interpretation:")
    if corr >= 0.80:
        print(f"    ρ ≥ 0.80 → {PASS}  Strong rank ordering — model correctly identifies")
        print(f"              who is expensive vs cheap relative to each other.")
    elif corr >= 0.60:
        print(f"    ρ ≥ 0.60 → {WARN}  Moderate rank ordering — reasonable but can improve.")
    else:
        print(f"    ρ < 0.60 → {FAIL}  Weak rank ordering — model struggles with relative value.")

    return corr


# ─────────────────────────────────────────────────────────────────────────────
def test_5_risk_discounts(model, df):
    separator("TEST 5 — Risk Discount Logic (Busquets Factor)")
    # Fixed MV at £70m — we're testing risk adjustments only, not market value effects
    BASE_MV = 70
    base = predict(model, df, 27, 3.0, 20, BASE_MV)

    scenarios = [
        ("Short contract (<1.5yr) -> cheaper  [MV=£70m]",
         predict(model, df, 27, 1.0, 20, BASE_MV), base, "cheaper"),
        ("Age 33 -> cheaper than 27  [MV=£70m]",
         predict(model, df, 33, 3.0, 20, BASE_MV), base, "cheaper"),
        ("Heavily injured (150 days) -> cheaper  [MV=£70m]",
         predict(model, df, 27, 3.0, 150, BASE_MV), base, "cheaper"),
        ("Combined risk (old + short contract) -> biggest discount  [MV=£70m]",
         predict(model, df, 34, 0.5, 120, BASE_MV), base, "cheaper"),
    ]

    all_pass = True
    for label, risky_val, safe_val, expectation in scenarios:
        ok = risky_val < safe_val if expectation == "cheaper" else risky_val > safe_val
        diff = safe_val - risky_val
        pct  = diff / safe_val * 100
        status = PASS if ok else FAIL
        print(f"  {status}  {label}")
        print(f"           -> Risky: \u00a3{risky_val:.1f}m vs Baseline: \u00a3{safe_val:.1f}m  (delta \u00a3{diff:.1f}m / {pct:.1f}% discount)")
        if not ok:
            all_pass = False

    return all_pass


# ─────────────────────────────────────────────────────────────────────────────
def test_6_no_absurd_outputs(model, df):
    separator("TEST 6 — No Absurd Edge Case Outputs")
    edge_cases = [
        ("Goalkeeper, 38yo, 1 contract yr, very injured",  38, 0.5, 200, 5),
        ("18yo wonderkid, 6yr deal, fully fit, high MV",   18, 6.0,   0, 160),
        ("Average player, median everything",              25, 2.5,  15, 30),
        ("Extreme market value input (£200m)",             22, 5.0,   5, 200),
        ("Zero market value (unlikely but valid)",         25, 3.0,  10, 1),
    ]

    all_pass = True
    for label, age, contract, injuries, mkt_val in edge_cases:
        pred = predict(model, df, age, contract, injuries, mkt_val)
        is_finite    = np.isfinite(pred)
        is_non_neg   = pred >= 0
        is_sensible  = 0 <= pred <= 600  # no transfer has ever exceeded £600m
        ok = is_finite and is_non_neg and is_sensible
        status = PASS if ok else FAIL
        print(f"  {status}  {label}")
        print(f"           → Predicted: £{pred:.1f}m  (finite={is_finite}, ≥0={is_non_neg}, ≤600={is_sensible})")
        if not ok:
            all_pass = False

    return all_pass


# ─────────────────────────────────────────────────────────────────────────────
if __name__ == "__main__":
    print("\n" + "=" * 70)
    print("  FairValue Transfer Model — Validation Suite")
    print("=" * 70)

    model, df = load_model_and_data()

    health_ok   = test_1_model_health(model, df)
    results     = test_2_magnitude_sanity(model, df)
    mono_ok     = test_3_monotonicity(model, df)
    spearman_r  = test_4_ranking_accuracy(results)
    risk_ok     = test_5_risk_discounts(model, df)
    edge_ok     = test_6_no_absurd_outputs(model, df)

    separator("OVERALL VERDICT")
    tests = [
        ("Model Health",        health_ok),
        ("Monotonicity",        mono_ok),
        ("Risk Discount Logic", risk_ok),
        ("Edge Case Safety",    edge_ok),
        ("Ranking Accuracy",    spearman_r >= 0.60),
    ]
    passed = sum(1 for _, ok in tests if ok)
    for label, ok in tests:
        print(f"  {'✅' if ok else '❌'}  {label}")

    print(f"\n  Score: {passed}/{len(tests)} tests passed")
    if passed == len(tests):
        print("\n  🎯 PRODUCTION READY — This is not a toy project.")
    elif passed >= 3:
        print("\n  ⚠️  MOSTLY READY — Minor issues to investigate above.")
    else:
        print("\n  ❌ NOT READY — Model needs retraining or debugging.")
    print("=" * 70 + "\n")