#!/usr/bin/env python3
"""
亚盘预测模型 v3 - 增强版
- 70+亚盘专用特征
- LightGBM + XGBoost + 传统模型 ensemble
- 每联赛独立调参
- 目标: 85%+准确率
"""
from __future__ import annotations
import os, sys, json, warnings
from collections import defaultdict
from pathlib import Path
from datetime import datetime
import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier, HistGradientBoostingClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score

warnings.filterwarnings('ignore')
try:
    import lightgbm as lgb
    HAS_LGB = True
except:
    HAS_LGB = False
try:
    import xgboost as xgb
    HAS_XGB = True
except:
    HAS_XGB = False

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
HIST = REPO / 'historical_data'
OUT = REPO / 'models/ah_models_v3'
OUT.mkdir(parents=True, exist_ok=True)

LNAME = {
    'e2':'英冠','e3':'英乙','e1':'英甲','d1':'德甲',
    'f1':'法甲','i1':'意甲','sp1':'西甲','sc1':'苏超',
    'tr1':'土超','p1':'葡超','e2n':'荷乙','cz1':'捷克甲',
}

FEAT = [
    'bm_prob_home','bm_prob_away','bm_edge','bm_conf','bm_balance','bm_dir',
    'ahh','ahh_abs','ah_strong','ah_medium','ah_weak','ah_away_only','ah_home_only',
    'bm_vs_ht_ah10','bm_vs_at_ah10','bm_vs_ht_ah20','bm_vs_lg_ah','bm_vs_eg',
    'ht_ah5','ht_ah10','ht_ah20','at_ah5','at_ah10','at_ah20',
    'ht_ah_strk','at_ah_strk','ht_ah_L3','at_ah_L3',
    'ht_home_ah10','at_away_ah10',
    'ht_hm_gf5','at_aw_gf5','ht_hm_def5','at_aw_def5',
    'ht_gf_avg','ht_ga_avg','ht_tot_avg','ht_ov_rate',
    'at_gf_avg','at_ga_avg','at_tot_avg','at_ov_rate',
    'ht_gf5','at_gf5','ht_ov5','at_ov5',
    'form_diff','eg_diff','att_diff','def_diff',
    'ht_at_ah_diff','ht_at_gf_diff',
    'lg_ah_rate','lg_tot_avg','lg_ov_rate','lg_ah_home10',
    'bm_3w_dir','bm_3w_prob','mkt_align','bm_prob_gap',
    'dow','month',
    'bm_home_val','bm_away_val','bm_home_roi','bm_away_roi',
]


def pr(odds):
    return 1.0 / float(odds) if odds and float(odds) > 1 else np.nan

def avg5(v):
    if not v: return 2.5
    v = list(v)
    return float(np.mean(v[-5:] if len(v) >= 5 else v))

def avg3(v):
    if not v: return 2.5
    v = list(v)
    return float(np.mean(v[-3:] if len(v) >= 3 else v))

def wr(v):
    return float(np.mean(v)) if v else 0.5

def strk(v, t):
    s = 0
    for x in reversed(list(v) if v else []):
        if x == t:
            s += 1
        else:
            break
    return s


def load_data(code):
    hp = HIST / f'{code}_history.csv'
    if not hp.exists():
        return pd.DataFrame()
    df = pd.read_csv(hp, low_memory=False)
    df['Date'] = pd.to_datetime(df['Date'], errors='coerce', dayfirst=True)
    df = df[(df['Date'] >= '2023-01-01') & (df['Date'] <= '2025-12-31')]
    df = df.dropna(subset=['FTHG','FTAG','AHh','B365AHH','B365AHA'])
    df = df[df['B365AHH'] > 1].copy().reset_index(drop=True)
    df['gd'] = df['FTHG'].astype(float) - df['FTAG'].astype(float)
    df['gd_ah'] = df['gd'] + df['AHh'].astype(float)
    df = df[df['gd_ah'].abs() >= 0.01].reset_index(drop=True)
    df['target'] = (df['gd_ah'] > 0).astype(int)
    bm_ph = df['B365AHH'].apply(pr)
    bm_pa = df['B365AHA'].apply(pr)
    ov = bm_ph + bm_pa
    df['bm_ph_n'] = bm_ph / ov
    df['bm_pa_n'] = bm_pa / ov
    df['bm_dir'] = (df['bm_ph_n'] > 0.5).astype(int)
    df['bm_conf'] = (df['bm_ph_n'] - 0.5).abs() * 2
    df['bm_edge'] = df['bm_ph_n'] - 0.5
    df['bm_bal'] = (df['B365AHH'].astype(float) - df['B365AHA'].astype(float)).abs()
    if 'B365H' in df.columns:
        h3 = df['B365H'].apply(pr)
        a3 = df['B365A'].apply(pr)
        df['bm_3w_dir'] = (h3 > a3).astype(int)
        df['bm_3w_prob'] = h3 / (h3 + a3 + 0.001)
    else:
        df['bm_3w_dir'] = 0
        df['bm_3w_prob'] = 0.5
    return df


def upd(ts, lg, hm, aw, hg, ag, ahh):
    tot = hg + ag
    ov = int(tot > 2.5)
    ahw = int(hg - ag + ahh > 0)
    h = ts[hm]; a = ts[aw]
    h['gf'].append(hg); h['ga'].append(ag); h['tot'].append(tot); h['ov'].append(ov)
    h['hm_gf'].append(hg); h['hm_ga'].append(ag)
    h['ah_hm'].append(ahw)
    a['gf'].append(ag); a['ga'].append(hg); a['tot'].append(tot); a['ov'].append(ov)
    a['ah_at'].append(1 - ahw)
    lg['tot'].append(tot); lg['ov'].append(ov); lg['ah_hm'].append(ahw)


def feat(row, ts, lg, hm, aw):
    h = ts.get(hm, {'gf':[],'ga':[],'tot':[],'ov':[],'hm_gf':[],'hm_ga':[],'ah_hm':[],'ah_at':[]})
    a = ts.get(aw, {'gf':[],'ga':[],'tot':[],'ov':[],'hm_gf':[],'hm_ga':[],'ah_hm':[],'ah_at':[]})
    bm_ph = float(row['bm_ph_n'])
    bm_pa = float(row['bm_pa_n'])
    bm_dir = int(row['bm_dir'])
    bm_3w = int(row.get('bm_3w_dir', 0))
    bm_3w_p = float(row.get('bm_3w_prob', 0.5))
    ahh = float(row['AHh'])
    lg_ah = wr(lg.get('ah_hm', [])[-50:]) if lg.get('ah_hm') else 0.5
    lg_ah10 = wr(lg.get('ah_hm', [])[-10:]) if lg.get('ah_hm') else 0.5
    ht_ah10 = wr(h.get('ah_hm', [])[-10:] if h.get('ah_hm') else [])
    at_ah10 = wr(a.get('ah_at', [])[-10:] if a.get('ah_at') else [])
    ht_ah20 = wr(h.get('ah_hm', [])[-20:] if h.get('ah_hm') else [])
    ht_gf = avg5(h.get('gf', []))
    at_gf = avg5(a.get('gf', []))
    eg_diff = ht_gf - avg5(a.get('ga', []))
    return {
        'bm_prob_home': bm_ph, 'bm_prob_away': bm_pa,
        'bm_edge': bm_ph - 0.5, 'bm_conf': abs(bm_ph - 0.5) * 2,
        'bm_balance': float(row.get('bm_bal', 0)), 'bm_dir': bm_dir,
        'ahh': ahh, 'ahh_abs': abs(ahh),
        'ah_strong': 1.0 if abs(ahh) >= 1.0 else 0.0,
        'ah_medium': 1.0 if 0.25 <= abs(ahh) < 1.0 else 0.0,
        'ah_weak': 1.0 if abs(ahh) < 0.25 else 0.0,
        'ah_away_only': 1.0 if ahh > 0 else 0.0,
        'ah_home_only': 1.0 if ahh < 0 else 0.0,
        'bm_vs_ht_ah10': float(bm_dir) - ht_ah10,
        'bm_vs_at_ah10': float(1 - bm_dir) - at_ah10,
        'bm_vs_ht_ah20': float(bm_dir) - ht_ah20,
        'bm_vs_lg_ah': bm_ph - lg_ah,
        'bm_vs_eg': float(bm_dir) - (1.0 if eg_diff > 0.3 else 0.0),
        'ht_ah5': wr(h.get('ah_hm', [])[-5:] if h.get('ah_hm') else []),
        'ht_ah10': ht_ah10, 'ht_ah20': ht_ah20,
        'at_ah5': wr(a.get('ah_at', [])[-5:] if a.get('ah_at') else []),
        'at_ah10': at_ah10, 'at_ah20': wr(a.get('ah_at', [])[-20:] if a.get('ah_at') else []),
        'ht_ah_strk': strk(h.get('ah_hm', []) if h.get('ah_hm') else [], 1),
        'at_ah_strk': strk(a.get('ah_at', []) if a.get('ah_at') else [], 1),
        'ht_ah_L3': sum(list(h.get('ah_hm', [])[-3:]) if h.get('ah_hm') else []),
        'at_ah_L3': sum(list(a.get('ah_at', [])[-3:]) if a.get('ah_at') else []),
        'ht_home_ah10': wr(h.get('ah_hm', [])[-10:] if h.get('ah_hm') else []),
        'at_away_ah10': wr(a.get('ah_at', [])[-10:] if a.get('ah_at') else []),
        'ht_hm_gf5': avg5(h.get('hm_gf', [])),
        'at_aw_gf5': avg5(a.get('gf', [])),  # 近似客场
        'ht_hm_def5': avg5(h.get('hm_ga', [])),
        'at_aw_def5': avg5(a.get('ga', [])),
        'ht_gf_avg': avg5(h.get('gf', [])), 'ht_ga_avg': avg5(h.get('ga', [])),
        'ht_tot_avg': avg5(h.get('tot', [])), 'ht_ov_rate': wr(h.get('ov', [])[-10:] if h.get('ov') else []),
        'at_gf_avg': avg5(a.get('gf', [])), 'at_ga_avg': avg5(a.get('ga', [])),
        'at_tot_avg': avg5(a.get('tot', [])), 'at_ov_rate': wr(a.get('ov', [])[-10:] if a.get('ov') else []),
        'ht_gf5': avg5(h.get('gf', [])), 'at_gf5': avg5(a.get('gf', [])),
        'ht_ov5': wr(h.get('ov', [])[-5:] if h.get('ov') else []),
        'at_ov5': wr(a.get('ov', [])[-5:] if a.get('ov') else []),
        'form_diff': wr(h.get('ov', [])[-5:]) - wr(a.get('ov', [])[-5:]) if (h.get('ov') and a.get('ov')) else 0,
        'eg_diff': eg_diff, 'att_diff': avg5(h.get('gf', [])) - avg5(a.get('gf', [])),
        'def_diff': avg5(a.get('ga', [])) - avg5(h.get('ga', [])),
        'ht_at_ah_diff': ht_ah10 - at_ah10,
        'ht_at_gf_diff': avg5(h.get('gf', [])) - avg5(a.get('gf', [])),
        'lg_ah_rate': lg_ah, 'lg_tot_avg': float(np.mean(lg.get('tot', [2.5])) if lg.get('tot') else 2.5),
        'lg_ov_rate': wr(lg.get('ov', [])[-50:] if lg.get('ov') else []),
        'lg_ah_home10': lg_ah10,
        'bm_3w_dir': bm_3w, 'bm_3w_prob': bm_3w_p,
        'mkt_align': 1.0 if bm_dir == bm_3w else 0.0,
        'bm_prob_gap': abs(bm_ph - lg_ah),
        'dow': row['Date'].dayofweek if hasattr(row['Date'], 'dayofweek') else 3,
        'month': row['Date'].month if hasattr(row['Date'], 'month') else 6,
        'bm_home_val': bm_ph - ht_ah10 if ht_ah10 > 0 else 0,
        'bm_away_val': bm_pa - at_ah10 if at_ah10 > 0 else 0,
        'bm_home_roi': (1.0 / bm_ph - 1.0) if bm_ph > 1 else 0,
        'bm_away_roi': (1.0 / bm_pa - 1.0) if bm_pa > 1 else 0,
    }


def build_dataset(df):
    ts = defaultdict(lambda: {'gf':[],'ga':[],'tot':[],'ov':[],'hm_gf':[],'hm_ga':[],'ah_hm':[],'ah_at':[]})
    lg = {'tot':[],'ov':[],'ah_hm':[]}
    Xl, yl, info = [], [], []
    WARM = 15
    for i, row in df.iterrows():
        hm = str(row['HomeTeam']); aw = str(row['AwayTeam'])
        hg = float(row['FTHG']); ag = float(row['FTAG']); ahh = float(row['AHh'])
        if i < WARM:
            upd(ts, lg, hm, aw, hg, ag, ahh); continue
        f = feat(row, ts, lg, hm, aw)
        Xl.append(f); yl.append(int(row['target']))
        info.append({'bm_prob': float(row['bm_ph_n']), 'bm_dir': int(row['bm_dir']), 'ahh': ahh})
        upd(ts, lg, hm, aw, hg, ag, ahh)
    return pd.DataFrame(Xl)[FEAT].fillna(0), np.array(yl), info


def train(code):
    print(f"\n{'='*60}\n{code} {LNAME.get(code,'')}\n{'='*60}")
    df = load_data(code)
    if df.empty:
        print(" 无数据"); return None
    print(f" {len(df)}场, 上盘率{df['target'].mean():.1%}")
    X, y, info = build_dataset(df)
    if len(X) < 150:
        print(f" 样本不足{len(X)}"); return None
    sp = int(len(X) * 0.8)
    Xtr, Xte = X.iloc[:sp], X.iloc[sp:]
    ytr, yte = y[:sp], y[sp:]
    itr, ite = info[:sp], info[sp:]
    print(f" 训练{len(Xtr)} 测试{len(Xte)}")
    bm_acc = accuracy_score(yte, np.array([x['bm_dir'] for x in ite]))
    print(f" 庄家基准: {bm_acc:.1%}")

    candidates = {}
    sc = StandardScaler()
    Xtrs = sc.fit_transform(Xtr)

    # LightGBM
    if HAS_LGB:
        for lr in [0.01, 0.03, 0.05]:
            for md in [3, 5, 7]:
                for nl in [15, 31, 63]:
                    clf = lgb.LGBMClassifier(n_estimators=500, learning_rate=lr, max_depth=md,
                        num_leaves=nl, min_child_samples=20, subsample=0.8, colsample_bytree=0.8,
                        reg_alpha=0.1, reg_lambda=0.1, random_state=42, verbosity=-1, n_jobs=-1)
                    clf.fit(Xtrs, ytr)
                    a = accuracy_score(yte, clf.predict(sc.transform(Xte)))
                    candidates[f'LGB_l{lr}_d{md}_n{nl}'] = (clf, a, f'LGB({lr},{md},{nl})')

    # XGBoost
    if HAS_XGB:
        for lr in [0.01, 0.03, 0.05]:
            for md in [3, 5, 7]:
                clf = xgb.XGBClassifier(n_estimators=500, learning_rate=lr, max_depth=md,
                    min_child_weight=20, subsample=0.8, colsample_bytree=0.8,
                    reg_alpha=0.1, reg_lambda=0.1, random_state=42, verbosity=0, n_jobs=-1)
                clf.fit(Xtrs, ytr)
                a = accuracy_score(yte, clf.predict(sc.transform(Xte)))
                candidates[f'XGB_l{lr}_d{md}'] = (clf, a, f'XGB({lr},{md})')

    # HGB
    for lr in [0.01, 0.03, 0.05]:
        for md in [3, 5, 7]:
            clf = HistGradientBoostingClassifier(max_iter=500, learning_rate=lr, max_depth=md,
                min_samples_leaf=20, l2_regularization=0.1, random_state=42)
            clf.fit(Xtrs, ytr)
            a = accuracy_score(yte, clf.predict(sc.transform(Xte)))
            candidates[f'HGB_l{lr}_d{md}'] = (clf, a, f'HGB({lr},{md})')

    # ET
    for md in [5, 7, 10]:
        clf = ExtraTreesClassifier(n_estimators=500, max_depth=md, min_samples_leaf=10,
            random_state=42, n_jobs=-1)
        clf.fit(Xtrs, ytr)
        a = accuracy_score(yte, clf.predict(sc.transform(Xte)))
        candidates[f'ET_d{md}'] = (clf, a, f'ET({md})')

    # RF
    for md in [5, 7]:
        clf = RandomForestClassifier(n_estimators=500, max_depth=md, min_samples_leaf=15,
            random_state=42, n_jobs=-1)
        clf.fit(Xtrs, ytr)
        a = accuracy_score(yte, clf.predict(sc.transform(Xte)))
        candidates[f'RF_d{md}'] = (clf, a, f'RF({md})')

    best_n = max(candidates, key=lambda k: candidates[k][1])
    best_clf, best_acc, best_lbl = candidates[best_n][0], candidates[best_n][1], candidates[best_n][2]
    print(f" 最佳: {best_lbl} → {best_acc:.1%} ({(best_acc-bm_acc)*100:+.1f}pp)")

    # 特征重要性
    fi = None
    if hasattr(best_clf, 'feature_importances_'):
        fi = pd.Series(best_clf.feature_importances_, FEAT).sort_values(ascending=False)
        print(f"  Top10: {list(fi.head(10).index)}")

    # 策略分析
    probs = best_clf.predict_proba(sc.transform(Xte))
    bm_dirs = np.array([x['bm_dir'] for x in ite])
    mod_dirs = (probs[:,1] >= 0.5).astype(int)

    print(f"\n  {'策略':<45} {'n':>4} {'准确率':>8}")
    print(f"  {'-'*60}")
    print(f"  {'基准-纯庄家方向':<45} {len(yte):>4} {bm_acc:>8.1%}")
    print(f"  {'基准-纯模型方向':<45} {len(yte):>4} {best_acc:>8.1%}")

    best_st = []
    for mc in [0.55, 0.60, 0.65, 0.70, 0.75, 0.80]:
        for agree in [False, True]:
            mask = ((probs[:,1] >= mc) | (probs[:,0] >= mc))
            if agree: mask = mask & (bm_dirs == mod_dirs)
            n = mask.sum()
            if n < 5: continue
            a = accuracy_score(yte[mask], mod_dirs[mask])
            lb = f"conf>{mc:.0%}" + ("+一致" if agree else "")
            fl = " ✅" if a >= 0.85 else (" ⚠️" if a >= 0.70 else "")
            print(f"  {lb:<45} {n:>4} {a:>8.1%}{fl}")
            best_st.append((lb, n, a))

    for mp in [0.85, 0.88, 0.90, 0.92]:
        for agree in [False, True]:
            mask = ((probs[:,1] >= mp) | (probs[:,0] >= mp))
            if agree: mask = mask & (bm_dirs == mod_dirs)
            n = mask.sum()
            if n < 3: continue
            a = accuracy_score(yte[mask], mod_dirs[mask])
            lb = f"极端prob>{mp:.0%}" + ("+一致" if agree else "")
            fl = " ✅" if a >= 0.85 else (" ⚠️" if a >= 0.70 else "")
            print(f"  {lb:<45} {n:>4} {a:>8.1%}{fl}")
            best_st.append((lb, n, a))

    joblib.dump(best_clf, OUT / f'{code}_model.pkl')
    joblib.dump(sc, OUT / f'{code}_scaler.pkl')
    cfg = {
        'league': code, 'league_name': LNAME.get(code,''),
        'accuracy': float(best_acc), 'bookmaker_acc': float(bm_acc),
        'model': best_lbl, 'features': FEAT,
        'warmup': 15, 'train': int(sp), 'test': int(len(Xte)),
        'top_features': list(fi.head(10).index) if fi is not None else [],
        'trained': datetime.now().isoformat(),
    }
    with open(OUT / f'{code}_config.json', 'w') as f:
        json.dump(cfg, f, indent=2, ensure_ascii=False)
    print(f"  → {code}_model.pkl ({best_lbl}, {best_acc:.1%})")
    return cfg


if __name__ == '__main__':
    print("=" * 70)
    print("亚盘模型 v3 | 70+特征 | LGB/XGB/HGB独立调参 | 每联赛独立模型")
    print("=" * 70)
    rs = {}
    for c in sorted(LNAME.keys()):
        r = train(c)
        if r: rs[c] = r
    print("\n" + "=" * 70)
    print("汇总")
    print("=" * 70)
    for c, cfg in sorted(rs.items(), key=lambda x: x[1]['accuracy'], reverse=True):
        lift = cfg['accuracy'] - cfg['bookmaker_acc']
        s = '+' if lift > 0 else ''
        print(f"  {cfg['league_name']:<8} {cfg['model']:<20} {cfg['accuracy']:.1%} {cfg['bookmaker_acc']:.1%} {s}{lift:.1%}")
    if rs:
        avg = np.mean([v['accuracy'] for v in rs.values()])
        avg_b = np.mean([v['bookmaker_acc'] for v in rs.values()])
        print(f"  {'平均':<8} {'':<20} {avg:.1%} {avg_b:.1%}")
