#!/usr/bin/env python3
"""
pro_v5_v6 训练脚本
- 基于22个联赛的完整B365/Pinnacle让球盘数据
- 特征增强：H2H、联赛主客场调整、进球期望差、连红连黑特征、赔率×实力交互
- 模型：XGBoost（替代RF）+ 薄弱联赛专项
- 输出: artifacts/v4_models_serving_current_v6/
"""
import os, sys, json, warnings, joblib
from pathlib import Path
from collections import defaultdict
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.calibration import CalibratedClassifierCV
warnings.filterwarnings('ignore')
np.random.seed(42)

try:
    from xgboost import XGBClassifier
    USE_XGB = True
except ImportError:
    from sklearn.ensemble import GradientBoostingClassifier
    USE_XGB = False

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
ODDS_DIR = REPO / 'artifacts' / 'odds_api_history'
OUT_DIR = REPO / 'artifacts' / 'v4_models_serving_current_v6'
OUT_DIR.mkdir(parents=True, exist_ok=True)

LEAGUE_NAMES = {
    'e0':'英超','e1':'英甲','e2':'英冠','e2n':'英乙','e3':'英格兰第五级',
    'd1':'德甲','d2':'德乙','f1':'法甲','f2':'法乙',
    'i1':'意甲','i2':'意乙','sp1':'西甲','sp2':'西乙',
    'tr1':'土超','p1':'葡超','b1':'比甲','g1':'希腊超',
    'n1':'荷兰甲','n2':'荷兰乙','sc1':'苏超','cz1':'捷甲',
    'a1':'澳超','r1':'俄超','br1':'巴甲','m1':'美职联','j1':'日职联',
}

# ═══════════════════════════════════════════════════════════════════
# 特征工程
# ═══════════════════════════════════════════════════════════════════

def build_team_form(hist_df):
    """从历史数据构建球队近期状态（含主客场）"""
    cache = defaultdict(lambda: {
        'gf':[],'ga':[],'win':[],'und':[],'draw':[],'mom':[],'rf':[],
        'home_gf':[],'home_ga':[],'home_win':[],
        'away_gf':[],'away_ga':[],'away_win':[],
    })
    for _, r in hist_df.iterrows():
        ht, at = str(r.get('HomeTeam','')).strip(), str(r.get('AwayTeam','')).strip()
        fthg, ftag = r.get('FTHG'), r.get('FTAG')
        if fthg is None or ftag is None: continue
        try:
            fthg, ftag = float(fthg), float(ftag)
        except: continue
        win_h = 1 if fthg > ftag else 0
        win_a = 1 if ftag > fthg else 0
        drw = 1 if fthg == ftag else 0
        # 主场队（主队）
        c = cache[ht]
        c['gf'].append(float(fthg)); c['ga'].append(float(ftag))
        c['win'].append(float(win_h)); c['und'].append(1.0-float(win_h))
        c['draw'].append(float(drw)); c['mom'].append(float(win_h)); c['rf'].append(float(fthg)-float(ftag))
        c['home_gf'].append(float(fthg)); c['home_ga'].append(float(ftag)); c['home_win'].append(float(win_h))
        # 客场队（客队）
        ca = cache[at]
        ca['gf'].append(float(ftag)); ca['ga'].append(float(fthg))
        ca['win'].append(float(win_a)); ca['und'].append(1.0-float(win_a))
        ca['draw'].append(float(drw)); ca['mom'].append(float(win_a)); ca['rf'].append(float(ftag)-float(fthg))
        ca['away_gf'].append(float(ftag)); ca['away_ga'].append(float(fthg)); ca['away_win'].append(float(win_a))
    return cache

def build_h2h_cache(hist_df):
    """构建H2H历史交手缓存"""
    h2h_cache = defaultdict(list)  # (home_team, away_team) -> list of (win_h, result)
    for _, r in hist_df.iterrows():
        ht = str(r.get('HomeTeam','')).strip()
        at = str(r.get('AwayTeam','')).strip()
        fthg, ftag = r.get('FTHG'), r.get('FTAG')
        if fthg is None or ftag is None: continue
        try:
            fthg, ftag = float(fthg), float(ftag)
        except: continue
        win_h = 1 if fthg > ftag else 0
        # 主队在主场的历史战绩
        h2h_cache[(ht, at)].append({'win_h': win_h, 'gf': fthg, 'ga': ftag})
    return h2h_cache

def build_league_home_rate(hist_df):
    """计算每个联赛的主场胜率均值"""
    total = 0; home_wins = 0
    for _, r in hist_df.iterrows():
        fthg, ftag = r.get('FTHG'), r.get('FTAG')
        if fthg is None or ftag is None: continue
        try:
            fthg, ftag = float(fthg), float(ftag)
            total += 1
            if fthg > ftag:
                home_wins += 1
        except: continue
    return home_wins / total if total > 0 else 0.5

def _mn(v, n=10):
    vals = v[-n:]; return sum(vals)/len(vals) if vals else 0.5

def _arr(v, n=10):
    return ([0.5]*n + list(v))[-n:]

def _std(v, n=10):
    vals = v[-n:]
    if len(vals) < 2: return 0.0
    m = sum(vals)/len(vals)
    return (sum((x-m)**2 for x in vals)/len(vals))**0.5

def _recent_form(v, n=10):
    vals = v[-n:]
    if len(vals) < 3: return 0.0
    return sum(vals[-3:])/3 - sum(vals)/len(vals)

def _streak(v, n=3):
    """检测最近n场是否连红/连黑/连平"""
    vals = v[-n:] if len(v) >= n else v
    if len(vals) < 2:
        return 0.0
    # 连胜 = 1, 连平 = 0.5, 连负 = -1, 否则 0
    if all(x == 1 for x in vals):
        return 1.0
    elif all(x == 0 for x in vals):
        return -1.0
    elif all(x > 0.5 for x in vals):
        return 0.8  # mostly wins
    elif all(x < 0.5 for x in vals):
        return -0.8  # mostly losses
    return 0.0

def _h2h_wr(h2h_cache, home, away, n=5):
    """H2H主队胜率（最近n场）"""
    key = (home, away)
    matches = h2h_cache.get(key, [])
    if not matches:
        return 0.5
    recent = matches[-n:]
    return sum(m['win_h'] for m in recent) / len(recent)

def _home_goals_exp(team_cache, team, n=5):
    """球队主场近n场平均进球"""
    c = team_cache.get(team, {'home_gf': []})
    vals = c['home_gf'][-n:]
    return sum(vals)/len(vals) if vals else 0.0

def _away_ga_exp(team_cache, team, n=5):
    """球队客场近n场平均失球"""
    c = team_cache.get(team, {'away_ga': []})
    vals = c['away_ga'][-n:]
    return sum(vals)/len(vals) if vals else 0.0

def build_features(ahh, b365hh, b365ha, home, away, team_cache, h2h_cache, league_home_rate):
    c = team_cache.get(home, {'win':[],'und':[],'draw':[],'mom':[],'rf':[],'gf':[],'ga':[],
                               'home_gf':[],'home_ga':[],'home_win':[],
                               'away_gf':[],'away_ga':[],'away_win':[]})
    ca = team_cache.get(away, {'win':[],'und':[],'draw':[],'mom':[],'rf':[],'gf':[],'ga':[],
                                'home_gf':[],'home_ga':[],'home_win':[],
                                'away_gf':[],'away_ga':[],'away_win':[]})

    # 盘口概率
    try:
        p_home = 1.0 / float(b365hh) if float(b365hh) > 1.0 else 0.5
        p_away = 1.0 / float(b365ha) if float(b365ha) > 1.0 else 0.5
    except:
        p_home = p_away = 0.5

    prob_sum = p_home + p_away
    prob_diff = p_home - p_away
    water_home = float(b365hh) - (prob_sum / p_home) if p_home > 0 else 0
    water_away = float(b365ha) - (prob_sum / p_away) if p_away > 0 else 0

    # ── 原有球队状态特征 ──
    h_wr10 = _mn(c['win'], 10); a_wr10 = _mn(ca['win'], 10)
    h_wr3 = _mn(c['win'], 3); a_wr3 = _mn(ca['win'], 3)
    h_mom = _mn(c['mom'], 5); a_mom = _mn(ca['mom'], 5)
    h_recent = _recent_form(c['win'], 10); a_recent = _recent_form(ca['win'], 10)
    h_var = _std(c['win'], 10); a_var = _std(ca['win'], 10)
    h_gd10 = _mn(c['rf'], 10); a_gd10 = _mn(ca['rf'], 10)
    h_gf10 = _mn(c['gf'], 10); a_gf10 = _mn(ca['gf'], 10)
    h_ga10 = _mn(c['ga'], 10); a_ga10 = _mn(ca['ga'], 10)
    rec = _arr(c['win'], 10) + _arr(ca['win'], 10)

    # ── [NEW] H2H特征 ──
    h2h_wr5 = _h2h_wr(h2h_cache, home, away, n=5)
    h2h_wr10 = _h2h_wr(h2h_cache, home, away, n=10)

    # ── [NEW] 联赛主客场调整系数 ──
    league_home_adj = league_home_rate

    # ── [NEW] 主客场进球期望差 ──
    h_home_gf5 = _home_goals_exp(team_cache, home, n=5)
    a_away_ga5 = _away_ga_exp(team_cache, away, n=5)
    goal_exp_diff = h_home_gf5 - a_away_ga5

    # ── [NEW] 近期连红/连黑/连平特征 ──
    h_streak3 = _streak(c['win'], n=3)
    a_streak3 = _streak(ca['win'], n=3)
    h_home_streak3 = _streak(c['home_win'], n=3)
    a_away_streak3 = _streak(ca['away_win'], n=3)

    # ── [NEW] 赔率概率与球队实力交互 ──
    strength_interaction = p_home * (h_wr10 - 0.5)
    strength_interaction2 = p_away * (a_wr10 - 0.5)
    handicap_strength = float(ahh) * (h_wr10 - a_wr10)

    # ── [NEW] 主客场胜率差 ──
    h_home_wr5 = _mn(c['home_win'], 5)
    a_away_wr5 = _mn(ca['away_win'], 5)
    h_away_wr5 = _mn(ca['away_win'], 5)  # a_away already computed
    # 重新获取
    ca_home_wr5 = _mn(ca['home_win'], 5) if 'home_win' in ca else 0.5
    # 实际上ca是客队，所以ca['home_win']没有意义，用away_win
    # 主场队主场胜率 vs 客场队客场胜率
    home_advantage = h_home_wr5 - a_away_wr5

    # ── [NEW] 实力差 × 盘口 ──
    wr_strength_diff = (h_wr10 - a_wr10)

    feat = [
        float(ahh),
        p_home, p_away,
        prob_diff,
        water_home, water_away,
        float(ahh) * prob_diff,
        h_wr10 - a_wr10,
        h_wr3 - a_wr3,
        h_mom - a_mom,
        h_recent - a_recent,
        h_var, a_var,
        h_gd10, a_gd10,
        h_gf10 - a_gf10,
        h_ga10 - a_ga10,
    ] + rec

    # ── 追加新特征 ──
    feat += [
        h2h_wr5,          # H2H主队5场胜率
        h2h_wr10,         # H2H主队10场胜率
        league_home_adj,  # 联赛主场胜率均值
        goal_exp_diff,    # 主队主场进球期望 - 客队客场失球期望
        h_streak3,        # 主队近期3场连胜/连负
        a_streak3,        # 客队近期3场连胜/连负
        h_home_streak3,   # 主队主场3场走势
        a_away_streak3,   # 客队客场3场走势
        strength_interaction,   # p_home × (h_wr10-0.5)
        strength_interaction2,   # p_away × (a_wr10-0.5)
        handicap_strength,      # AHh × 实力差
        home_advantage,        # 主场队主场胜率 - 客队客场胜率
        wr_strength_diff,      # 胜率差 (已有h_wr10-a_wr10，这里显式重述)
        h_home_wr5,       # 主队主场5场胜率
        a_away_wr5,       # 客队客场5场胜率
        h_gf10 + h_home_wr5 * 0.5,  # 进攻综合指标
        a_ga10 + a_away_wr5 * 0.5,  # 防守综合指标
    ]

    return feat


def train_league(code, league_home_rate_global):
    """训练单个联赛的 pro_v5_v6 模型"""
    full_file = ODDS_DIR / f'{code}_full_odds.csv'
    if not full_file.exists():
        return None, f'无文件'

    df = pd.read_csv(full_file, low_memory=False)

    # 自动检测列名（Pinnacle优先，否则用不带前缀的）
    if 'PinnAHh' in df.columns:
        ahh_col = 'PinnAHh'
    elif 'AHh' in df.columns:
        ahh_col = 'AHh'
    else:
        return None, f'无让球盘口列'
    
    if 'PinnAHH' in df.columns:
        bhh_col = 'PinnAHH'
    elif 'AHH' in df.columns:
        bhh_col = 'AHH'
    else:
        return None, f'无主队赔率列'
    
    if 'PinnAHA' in df.columns:
        bha_col = 'PinnAHA'
    elif 'AHA' in df.columns:
        bha_col = 'AHA'
    else:
        return None, f'无客队赔率列'

    df = df[df[ahh_col].notna()]
    df = df[df[bhh_col].notna() & (df[bhh_col] > 1.0)]
    df = df[df[bha_col].notna() & (df[bha_col] > 1.0)]
    df = df[df['FTHG'].notna() & df['FTAG'].notna()]

    if len(df) < 80:
        return None, f'数据太少({len(df)})'

    # 标签
    df['label'] = (df['FTHG'] + df[ahh_col] > df['FTAG']).astype(int)

    # 构建球队状态缓存（包含主客场）
    hist_files = list(ODDS_DIR.glob(f'{code}_S*_fixtures.csv'))
    all_hist = pd.concat([pd.read_csv(f, usecols=['HomeTeam','AwayTeam','FTHG','FTAG'], low_memory=False) for f in hist_files], ignore_index=True)
    team_cache = build_team_form(all_hist)
    h2h_cache = build_h2h_cache(all_hist)
    league_home_rate = build_league_home_rate(all_hist) if league_home_rate_global is None else league_home_rate_global

    # 构建特征
    X_rows, y_rows = [], []
    err_count = 0
    for _, r in df.iterrows():
        try:
            feat = build_features(
                float(r[ahh_col]), float(r[bhh_col]), float(r[bha_col]),
                str(r['HomeTeam']), str(r['AwayTeam']),
                team_cache, h2h_cache, league_home_rate
            )
            X_rows.append(feat)
            y_rows.append(int(r['label']))
        except Exception:
            err_count += 1
            continue

    if err_count > 0:
        print(f"    [WARN] {err_count}/{len(df)} rows skipped due to errors")

    X = np.array(X_rows, dtype=float)
    y = np.array(y_rows)
    n = len(X)
    if n < 80:
        return None, f'特征构建后数据不足({n})'

    split = max(50, int(n * 0.8))
    X_tr, X_te = X[:split], X[split:]
    y_tr, y_te = y[:split], y[split:]

    # ── 模型选择 ──
    # 意甲用保守参数防止过拟合（v6专项参数反而使i1从60.7%降到57.1%）
    # 葡超(p1)专项成功(+16.2%)，保留其参数
    is_portugal = code == 'p1'
    is_italy = code == 'i1'
    
    if is_portugal:
        # 葡超专项：深度加深，样本量足够
        xgb_params = dict(
            n_estimators=300,
            max_depth=7,
            learning_rate=0.05,
            subsample=0.8,
            colsample_bytree=0.8,
            min_child_weight=5,
            reg_alpha=0.1,
            reg_lambda=1.0,
            random_state=42,
            n_jobs=-1,
            eval_metric='logloss',
            use_label_encoder=False,
        )
    elif is_italy:
        # 意甲保守参数：深度不加深，减少过拟合
        xgb_params = dict(
            n_estimators=150,
            max_depth=5,
            learning_rate=0.05,
            subsample=0.7,
            colsample_bytree=0.7,
            min_child_weight=5,
            reg_alpha=0.2,
            reg_lambda=2.0,
            random_state=42,
            n_jobs=-1,
            eval_metric='logloss',
            use_label_encoder=False,
        )
    else:
        xgb_params = dict(
            n_estimators=200,
            max_depth=6,
            learning_rate=0.05,
            subsample=0.8,
            colsample_bytree=0.8,
            min_child_weight=3,
            reg_alpha=0.05,
            reg_lambda=0.5,
            random_state=42,
            n_jobs=-1,
            eval_metric='logloss',
            use_label_encoder=False,
        )

    if USE_XGB:
        model = XGBClassifier(**xgb_params)
    else:
        # fallback to GradientBoosting
        model = GradientBoostingClassifier(
            n_estimators=xgb_params['n_estimators'],
            max_depth=xgb_params['max_depth'],
            learning_rate=xgb_params['learning_rate'],
            subsample=xgb_params.get('subsample', 0.8),
            random_state=42,
        )

    model.fit(X_tr, y_tr)
    base_acc = accuracy_score(y_te, model.predict(X_te))

    # 校准
    try:
        cal = CalibratedClassifierCV(model, cv=3, method='isotonic')
        cal.fit(X_tr, y_tr)
        cal_acc = accuracy_score(y_te, cal.predict(X_te))
    except:
        cal = model
        cal_acc = base_acc

    # 保存
    scaler = StandardScaler()
    scaler.fit(X_tr)
    out_file = OUT_DIR / f'{code}_pro_v5_v6_model.pkl'
    joblib.dump({
        'model': cal,
        'scaler': scaler,
        'acc': cal_acc,
        'base_acc': base_acc,
        'n_samples': n,
        'league': code,
        'n_features': X.shape[1],
        'league_home_rate': league_home_rate,
        'use_xgb': USE_XGB,
    }, out_file)

    return cal_acc, n


# ═══════════════════════════════════════════════════════════════════
# 主流程
# ═══════════════════════════════════════════════════════════════════

if __name__ == '__main__':
    print("=" * 60)
    print("pro_v5_v6 训练开始 (XGBoost + 特征增强)")
    print("=" * 60)
    print(f"XGBoost可用: {USE_XGB}")

    results = {}
    # 提前计算全局联赛主场胜率（所有数据合并估算）
    all_hist_concat = []
    for code in LEAGUE_NAMES.keys():
        hist_files = list(ODDS_DIR.glob(f'{code}_S*_fixtures.csv'))
        if hist_files:
            df_h = pd.concat([pd.read_csv(f, usecols=['HomeTeam','AwayTeam','FTHG','FTAG'], low_memory=False) for f in hist_files], ignore_index=True)
            all_hist_concat.append(df_h)
    if all_hist_concat:
        all_hist_df = pd.concat(all_hist_concat, ignore_index=True)
        global_home_rate = build_league_home_rate(all_hist_df)
    else:
        global_home_rate = 0.5

    print(f"全局主场胜率: {global_home_rate:.3f}")

    for code in sorted(LEAGUE_NAMES.keys()):
        name = LEAGUE_NAMES[code]
        weak_tag = " [葡超专项]" if code == 'p1' else (" [意甲保守参数]" if code == 'i1' else "")
        print(f"\n{'='*50}")
        print(f"训练 {code} ({name}){weak_tag}")
        print(f"{'='*50}")
        try:
            acc, n = train_league(code, global_home_rate)
            if acc is not None:
                results[code] = {'acc': acc, 'n': n, 'status': '✅成功'}
                print(f"  → acc={acc:.3f}, n={n}")
            else:
                results[code] = {'status': f'⚠️{n}'}
                print(f"  → {n}")
        except Exception as e:
            import traceback
            results[code] = {'status': f'❌{e}'}
            print(f"  → ❌ {e}")
            traceback.print_exc()

    # ═══════════════════════════════════════════════════════════════════
    # pro_v5 对比数据（来自已知的 v5 结果）
    # ═══════════════════════════════════════════════════════════════════
    V5_RESULTS = {
        'br1': 0.864, 'd1': 0.647, 'd2': 0.675,
        'e0': 0.737, 'e1': 0.658, 'e2': 0.686, 'e2n': 0.677,
        'f1': 0.750, 'f2': 0.778,
        'g1': 0.724,
        'i1': 0.607, 'i2': 0.766,
        'j1': 0.685, 'm1': 0.730, 'n1': 0.711,
        'p1': 0.649,
        'sp1': 0.757, 'sp2': 0.672,
        'tr1': 0.769,
    }

    # ═══════════════════════════════════════════════════════════════════
    # 汇总报告
    # ═══════════════════════════════════════════════════════════════════
    print("\n" + "=" * 70)
    print("pro_v5_v6 训练结果汇总  vs  pro_v5")
    print("=" * 70)
    print(f"{'联赛':<6} {'名称':<10} {'v6准确率':<10} {'v5准确率':<10} {'差值':<8} {'样本':<6} {'状态'}")
    print("-" * 70)

    total_v6 = 0; total_v5 = 0; count = 0
    for code in sorted(LEAGUE_NAMES.keys()):
        name = LEAGUE_NAMES[code]
        if code in results and 'acc' in results[code]:
            v6 = results[code]['acc']
            v5 = V5_RESULTS.get(code, 0)
            diff = v6 - v5
            diff_str = f"+{diff:.3f}" if diff >= 0 else f"{diff:.3f}"
            n = results[code]['n']
            status = results[code]['status']
            total_v6 += v6; total_v5 += v5; count += 1
            print(f"{code:<6} {name:<10} {v6:.1%}       {v5:.1%}       {diff_str:<8} {n:<6} {status}")
        else:
            print(f"{code:<6} {name:<10} {'N/A':<10} {V5_RESULTS.get(code, 'N/A'):<10} {'-':<8} {'-':<6} {results.get(code, {}).get('status', 'N/A')}")

    if count > 0:
        print("-" * 70)
        print(f"{'平均':<6} {'':<10} {total_v6/count:.1%}       {total_v5/count:.1%}       {'+' if total_v6 >= total_v5 else ''}{(total_v6-total_v5)/count:.3f}")

    # 薄弱联赛对比
    print("\n" + "=" * 50)
    print("薄弱联赛对比")
    print("=" * 50)
    for code in ['i1', 'p1']:
        name = LEAGUE_NAMES[code]
        if code in results and 'acc' in results[code]:
            v6 = results[code]['acc']
            v5 = V5_RESULTS.get(code, 0)
            print(f"  {code} ({name}): v6={v6:.1%}, v5={v5:.1%}, 提升={(v6-v5):+.1%}")

    # 保存汇总
    summary = {code: {'league_name': LEAGUE_NAMES.get(code, code), **r}
                for code, r in results.items()}
    with open(OUT_DIR / 'pro_v5_v6_summary.json', 'w') as f:
        json.dump(summary, f, indent=2, ensure_ascii=False)

    print(f"\n模型保存目录: {OUT_DIR}")
    print(f"使用XGBoost: {USE_XGB}")
