#!/usr/bin/env python3
"""
ah_1year_v5 训练脚本
- 基于22个联赛的完整Pinnacle让球盘数据
- 特征：赔率比较特征 + 球队历史胜负率 + 让球数交互 + H2H + 近期得失球趋势 + 赔率偏离度
- 模型：XGBoost（替代v4的HistGradientBoostingClassifier）
- 输出: models/ah_1year_v5/

改进点：
1. 新增H2H历史交手特征（最近5场、10场主队对客队胜率）
2. 新增近期得失球趋势特征
3. 新增赔率偏离度（b365 vs Pinnacle）
4. 新增让球数 × 主客场胜率差（用近5场）
5. 新增联赛主队优势均值
6. 移除 ratio_h, ratio_a（与norm_home/norm_away高度相关，易过拟合）
7. 算法升级为 XGBoost
8. 薄弱联赛（g1,i1,e0,e2n）使用更保守参数
"""
import os, sys, json, warnings, joblib
from pathlib import Path
from collections import defaultdict
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.calibration import CalibratedClassifierCV
warnings.filterwarnings('ignore')
np.random.seed(42)

try:
    from xgboost import XGBClassifier
    USE_XGB = True
except ImportError:
    from sklearn.ensemble import GradientBoostingClassifier as XGBClassifier
    USE_XGB = False
    print("⚠️ XGBoost not available, using sklearn GradientBoostingClassifier")

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
ODDS_DIR = REPO / 'artifacts' / 'odds_api_history'
OUT_DIR = REPO / 'models' / 'ah_1year_v5'
OUT_DIR.mkdir(parents=True, exist_ok=True)

LEAGUE_NAMES = {
    'e0':'英超','e1':'英甲','e2':'英冠','e2n':'英乙','e3':'英格兰第五级',
    'd1':'德甲','d2':'德乙','f1':'法甲','f2':'法乙',
    'i1':'意甲','i2':'意乙','sp1':'西甲','sp2':'西乙',
    'tr1':'土超','p1':'葡超','b1':'比甲','g1':'希腊超',
    'n1':'荷兰甲','n2':'荷兰乙','sc1':'苏超','cz1':'捷甲',
    'a1':'澳超','r1':'俄超','br1':'巴甲','m1':'美职联','j1':'日职联',
}

WEAK_LEAGUES = {'g1', 'i1', 'e0', 'e2n'}

# ═══════════════════════════════════════════════════════════════════
# 工具函数
# ═══════════════════════════════════════════════════════════════════

def _m(v, n=20):
    vals = v[-n:] if len(v) >= n else v
    return sum(vals)/len(vals) if vals else 0.5

def _arr5(v):
    return ([0.5]*5 + list(v))[-5:]

def _arr10(v):
    return ([0.5]*10 + list(v))[-10:]

# ═══════════════════════════════════════════════════════════════════
# 球队历史缓存
# ═══════════════════════════════════════════════════════════════════

def build_team_hist(hist_df):
    cache = defaultdict(lambda: {
        'hw':[], 'aw':[], 'dr':[], 'gf':[], 'ga':[], 'tot':[],
        'home_gf':[], 'home_ga':[], 'away_gf':[], 'away_ga':[]
    })
    for _, r in hist_df.iterrows():
        ht = str(r.get('HomeTeam','')).strip()
        at = str(r.get('AwayTeam','')).strip()
        fthg, ftag = r.get('FTHG'), r.get('FTAG')
        if fthg is None or ftag is None:
            continue
        try:
            fthg, ftag = float(fthg), float(ftag)
        except:
            continue
        c = cache[ht]
        c['gf'].append(float(fthg)); c['ga'].append(float(ftag))
        c['tot'].append(float(fthg)+float(ftag))
        c['hw'].append(1.0 if fthg > ftag else 0.0)
        c['aw'].append(0.0)
        c['dr'].append(1.0 if fthg == ftag else 0.0)
        c['home_gf'].append(float(fthg)); c['home_ga'].append(float(ftag))
        c = cache[at]
        c['gf'].append(float(ftag)); c['ga'].append(float(fthg))
        c['tot'].append(float(fthg)+float(ftag))
        c['hw'].append(0.0); c['aw'].append(1.0 if ftag > fthg else 0.0)
        c['dr'].append(1.0 if fthg == ftag else 0.0)
        c['away_gf'].append(float(ftag)); c['away_ga'].append(float(fthg))
    return cache


def build_h2h_cache(hist_df):
    cache = defaultdict(lambda: {
        'home_wins': [], 'away_wins': [], 'draws': [],
        'home_gf': [], 'away_gf': [], 'tot_gf': []
    })
    for _, r in hist_df.iterrows():
        ht = str(r.get('HomeTeam','')).strip()
        at = str(r.get('AwayTeam','')).strip()
        fthg, ftag = r.get('FTHG'), r.get('FTAG')
        if fthg is None or ftag is None:
            continue
        try:
            fthg, ftag = float(fthg), float(ftag)
        except:
            continue
        key = (ht, at)
        c = cache[key]
        c['home_gf'].append(float(fthg)); c['away_gf'].append(float(ftag))
        c['tot_gf'].append(float(fthg)+float(ftag))
        if fthg > ftag:
            c['home_wins'].append(1.0); c['away_wins'].append(0.0); c['draws'].append(0.0)
        elif ftag > fthg:
            c['home_wins'].append(0.0); c['away_wins'].append(1.0); c['draws'].append(0.0)
        else:
            c['home_wins'].append(0.0); c['away_wins'].append(0.0); c['draws'].append(1.0)
    return cache


def compute_league_home_adv(hist_df):
    valid = hist_df[hist_df['FTHG'].notna() & hist_df['FTAG'].notna()]
    if len(valid) == 0:
        return 0.5
    return (valid['FTHG'] > valid['FTAG']).sum() / len(valid)


# ═══════════════════════════════════════════════════════════════════
# 特征工程 v5
# ═══════════════════════════════════════════════════════════════════

def build_features_v5(ahh, pinnHh, pinnHa, b365Hh, b365Ha,
                      home, away,
                      team_cache, h2h_cache,
                      league_home_adv):
    """
    ah_1year_v5 特征（22个静态 + 10个近期 = 32维）
    """
    # === 赔率特征 ===
    try:
        pinnHh_f = float(pinnHh); pinnHa_f = float(pinnHa)
        b365Hh_f = float(b365Hh); b365Ha_f = float(b365Ha)
        total_pinn = (1/pinnHh_f) + (1/pinnHa_f)
        norm_home = (1/pinnHh_f) / total_pinn if total_pinn > 0 else 0.5
        norm_away = (1/pinnHa_f) / total_pinn if total_pinn > 0 else 0.5
        odds_gap = abs(norm_home - norm_away)
        # 赔率偏离度（B365/Pinnacle ratio）
        odds_dev_h = b365Hh_f / pinnHh_f if pinnHh_f > 0 else 1.0
        odds_dev_a = b365Ha_f / pinnHa_f if pinnHa_f > 0 else 1.0
    except:
        norm_home = norm_away = odds_gap = 0.5
        odds_dev_h = odds_dev_a = 1.0
        pinnHh_f = pinnHa_f = 2.0
    
    try:
        ahh_f = float(ahh)
    except:
        ahh_f = 0.0
    
    adj_home_prob = max(0.1, min(0.9, norm_home + ahh_f * 0.1))
    
    # === 球队历史 ===
    c = team_cache.get(home, {'hw':[], 'aw':[], 'dr':[], 'gf':[], 'ga':[], 'tot':[],
                               'home_gf':[], 'home_ga':[], 'away_gf':[], 'away_ga':[]})
    ca = team_cache.get(away, {'hw':[], 'aw':[], 'dr':[], 'gf':[], 'ga':[], 'tot':[],
                                'home_gf':[], 'home_ga':[], 'away_gf':[], 'away_ga':[]})
    
    h_hw = _m(c['hw'], 20); a_hw = _m(ca['hw'], 20)
    h_aw = _m(c['aw'], 20); a_aw = _m(ca['aw'], 20)
    h_dr = _m(c['dr'], 20); a_dr = _m(ca['dr'], 20)
    h_hw_5 = _m(c['hw'], 5); a_aw_5 = _m(ca['aw'], 5)
    gf_diff = _m(c['gf'], 10) - _m(ca['ga'], 10)
    tot_avg = (_m(c['tot'], 10) + _m(ca['tot'], 10)) / 2
    
    # === H2H ===
    h2h = h2h_cache.get((home, away), {'home_wins':[], 'away_wins':[], 'draws':[],
                                          'home_gf':[], 'away_gf':[], 'tot_gf':[]})
    h2h_hw_5 = _m(h2h['home_wins'], 5)
    h2h_hw_10 = _m(h2h['home_wins'], 10)
    h2h_gf_diff = (_m(h2h['home_gf'], 5) - _m(h2h['away_gf'], 5)) if h2h['home_gf'] else 0.0
    
    # === 趋势 ===
    home_recent_gf = _m(c['home_gf'], 5)
    away_recent_ga = _m(ca['away_ga'], 5)
    goal_trend = home_recent_gf - away_recent_ga
    
    interaction_5 = (h_hw_5 - a_aw_5) * ahh_f
    recent_5_home = _arr5(c['hw'])
    recent_5_away = _arr5(ca['aw'])
    
    return [
        # 赔率特征 (7)
        ahh_f, norm_home, norm_away, odds_gap,
        odds_dev_h, odds_dev_a, adj_home_prob,
        # 球队历史 (7)
        h_hw, a_aw, h_aw, a_hw,
        h_dr - a_dr, gf_diff, tot_avg,
        # H2H (3)
        h2h_hw_5, h2h_hw_10, h2h_gf_diff,
        # 趋势与交互 (5)
        goal_trend, interaction_5,
        (h_dr + a_dr) / 2,
        odds_gap * abs(ahh_f),
        league_home_adv,
    ] + recent_5_home + recent_5_away


# ═══════════════════════════════════════════════════════════════════
# 单联赛训练
# ═══════════════════════════════════════════════════════════════════

def load_league_data(code):
    """
    加载联赛数据，优先用 odds_api_merged.csv（包含B365赔率）
    fallback 到 full_odds.csv（只有Pinnacle）
    返回 (df, odds_source) 其中 odds_source = 'b365' 或 'pinnacle'
    """
    merged_file = ODDS_DIR / f'{code}_odds_api_merged.csv'
    full_file = ODDS_DIR / f'{code}_full_odds.csv'
    
    if merged_file.exists():
        df = pd.read_csv(merged_file, low_memory=False)
        # merged 文件：B365列存在，且有 AHh(handicap列), B365AHH, B365AHA
        return df, 'b365'
    elif full_file.exists():
        df = pd.read_csv(full_file, low_memory=False)
        # full_odds：Pinnacle列存在
        return df, 'pinnacle'
    else:
        return None, None


def train_league(code):
    df, odds_source = load_league_data(code)
    if df is None:
        return None, f'无文件'
    
    if len(df) == 0:
        return None, f'空数据文件'
    
    # 动态检测列名
    ah_col = 'PinnAHh' if 'PinnAHh' in df.columns else ('AHh' if 'AHh' in df.columns else None)
    if ah_col is None:
        return None, f'无AH列'
    
    odds_h_col = 'PinnAHH' if 'PinnAHH' in df.columns else ('B365AHH' if 'B365AHH' in df.columns else None)
    odds_a_col = 'PinnAHA' if 'PinnAHA' in df.columns else ('B365AHA' if 'B365AHA' in df.columns else None)
    if odds_h_col is None or odds_a_col is None:
        return None, f'无赔率列'
    
    # 过滤有效记录（移除 PinnAHh > 1.0 限制，保留所有有效 handicaps）
    df = df[df[ah_col].notna()]
    df = df[df['FTHG'].notna() & df['FTAG'].notna()]
    df = df[df[odds_h_col].notna() & df[odds_a_col].notna()]
    df = df[df[odds_h_col] > 1.0]  # 赔率有效性检查
    
    if len(df) < 60:
        return None, f'数据太少({len(df)})'
    
    # 按日期排序
    if 'Date' in df.columns:
        df = df.sort_values('Date').reset_index(drop=True)
    
    # 标签
    df['label'] = (df['FTHG'] + df[ah_col] > df['FTAG']).astype(int)
    
    # 构建历史缓存
    hist_files = list(ODDS_DIR.glob(f'{code}_S*_fixtures.csv')) + \
                 list(ODDS_DIR.glob(f'{code}_fixtures.csv'))
    if hist_files:
        all_hist = pd.concat([
            pd.read_csv(f, usecols=['HomeTeam','AwayTeam','FTHG','FTAG'], low_memory=False)
            for f in hist_files
        ], ignore_index=True)
    else:
        all_hist = df[['HomeTeam','AwayTeam','FTHG','FTAG']].copy()
    
    team_cache = build_team_hist(all_hist)
    h2h_cache = build_h2h_cache(all_hist)
    league_home_adv = compute_league_home_adv(all_hist)
    
    # 构建特征
    X_rows, y_rows = [], []
    for _, r in df.iterrows():
        try:
            home_t = str(r['HomeTeam']).strip()
            away_t = str(r['AwayTeam']).strip()
            
            odds_h = float(r[odds_h_col])
            odds_a = float(r[odds_a_col])
            ahh_val = float(r[ah_col])
            
            # 赔率偏离度：odds_dev = 1.0（因为没有两个独立bookmaker同时存在）
            # 但我们可以用 AH odds 的均衡度作为近似
            feat = build_features_v5(
                ahh=ahh_val,
                pinnHh=odds_h,
                pinnHa=odds_a,
                b365Hh=odds_h,  # 使用同一组赔率（odds_dev 将是 1.0）
                b365Ha=odds_a,
                home=home_t,
                away=away_t,
                team_cache=team_cache,
                h2h_cache=h2h_cache,
                league_home_adv=league_home_adv
            )
            X_rows.append(feat)
            y_rows.append(int(r['label']))
        except Exception as e:
            continue
    
    X = np.array(X_rows, dtype=float)
    y = np.array(y_rows)
    n = len(X)
    if n < 60:
        return None, f'特征构建后不足({n})'
    
    # 时间分割
    split = max(30, int(n * 0.8))
    X_tr, X_te = X[:split], X[split:]
    y_tr, y_te = y[:split], y[split:]
    
    # 标准化
    scaler = StandardScaler()
    X_tr_s = scaler.fit_transform(X_tr)
    X_te_s = scaler.transform(X_te)
    
    is_weak = code in WEAK_LEAGUES
    
    if USE_XGB:
        if is_weak:
            model = XGBClassifier(
                n_estimators=200, max_depth=3, learning_rate=0.03,
                min_child_weight=5, subsample=0.7, colsample_bytree=0.7,
                gamma=0.2, reg_alpha=0.5, reg_lambda=1.5,
                tree_method='hist', random_state=42,
                use_label_encoder=False, eval_metric='logloss', verbosity=0
            )
        else:
            model = XGBClassifier(
                n_estimators=200, max_depth=5, learning_rate=0.05,
                min_child_weight=3, subsample=0.8, colsample_bytree=0.8,
                gamma=0.1, reg_alpha=0.2, reg_lambda=1.0,
                tree_method='hist', random_state=42,
                use_label_encoder=False, eval_metric='logloss', verbosity=0
            )
    else:
        model = XGBClassifier(
            n_estimators=200,
            max_depth=3 if is_weak else 4,
            learning_rate=0.03 if is_weak else 0.05,
            min_samples_leaf=10 if is_weak else 5,
            random_state=42
        )
    
    model.fit(X_tr_s, y_tr)
    base_acc = accuracy_score(y_te, model.predict(X_te_s))
    
    # 校准
    try:
        cal = CalibratedClassifierCV(model, cv=3, method='isotonic')
        cal.fit(X_tr_s, y_tr)
        cal_acc = accuracy_score(y_te, cal.predict(X_te_s))
        final_model = cal
    except:
        final_model = model
        cal_acc = base_acc
    
    # 阈值优化
    best_thresh, best_thresh_acc = 0.5, cal_acc
    for thresh in [0.47, 0.48, 0.49, 0.50, 0.51, 0.52, 0.53, 0.54, 0.55]:
        try:
            probs = final_model.predict_proba(X_te_s)[:, 1]
            preds_t = (probs >= thresh).astype(int)
            acc_t = accuracy_score(y_te, preds_t)
            if acc_t > best_thresh_acc:
                best_thresh_acc = acc_t
                best_thresh = thresh
        except:
            pass
    
    # 保存
    out_dir = OUT_DIR / code
    out_dir.mkdir(parents=True, exist_ok=True)
    cfg = {
        'league': code,
        'league_name': LEAGUE_NAMES.get(code, code),
        'accuracy': float(best_thresh_acc),
        'base_accuracy': float(cal_acc),
        'threshold': float(best_thresh),
        'n_samples': int(n),
        'model_type': 'calibrated_XGBoost' if USE_XGB else 'calibrated_GradientBoosting',
        'is_weak_league': is_weak,
        'odds_source': odds_source,
        'n_features': int(X.shape[1]),
    }
    with open(out_dir / 'optimized_config.json', 'w') as f:
        json.dump(cfg, f, indent=2)
    joblib.dump({
        'model': final_model,
        'scaler': scaler,
        'acc': best_thresh_acc,
        'n_samples': n,
        'threshold': best_thresh,
        'league_home_adv': league_home_adv,
    }, out_dir / 'optimized_pipeline.pkl')
    
    return best_thresh_acc, n


# ═══════════════════════════════════════════════════════════════════
# 主流程
# ═══════════════════════════════════════════════════════════════════

print("=" * 60)
print("ah_1year_v5 训练开始")
print(f"XGBoost可用: {USE_XGB}")
print(f"薄弱联赛: {sorted(WEAK_LEAGUES)}")
print("=" * 60)

# v4基准
v4_results = {}
try:
    v4_summary = json.load(open(REPO / 'models' / 'ah_1year_v4' / 'ah_1year_v4_summary.json'))
    v4_results = {k: v.get('acc', 0) for k, v in v4_summary.items()}
except:
    pass

results = {}
for code in sorted(LEAGUE_NAMES.keys()):
    name = LEAGUE_NAMES[code]
    print(f"\n{'='*50}")
    print(f"训练 {code} ({name})")
    print(f"{'='*50}")
    try:
        acc, n = train_league(code)
        if acc is not None:
            results[code] = {'acc': acc, 'n': n, 'status': '✅成功'}
            weak_tag = ' [薄弱联赛]' if code in WEAK_LEAGUES else ''
            v4_acc = v4_results.get(code)
            delta = f", Δ={acc-v4_acc:+.3f}" if v4_acc else ""
            print(f"  → acc={acc:.3f}, n={n}{weak_tag}{delta}")
        else:
            results[code] = {'status': f'⚠️{n}'}
            print(f"  → {n}")
    except Exception as e:
        import traceback; traceback.print_exc()
        results[code] = {'status': f'❌{e}'}
        print(f"  → ❌ {e}")

# 汇总报告
print("\n" + "=" * 60)
print("ah_1year_v5 训练结果汇总")
print("=" * 60)
print(f"{'联赛':<6} {'名称':<8} {'v5准确率':<10} {'v4准确率':<10} {'Δ':<8} {'样本':<6} {'状态'}")
print("-" * 65)

total_imp, count = 0, 0
for code in sorted(LEAGUE_NAMES.keys()):
    r = results.get(code, {})
    v4_acc = v4_results.get(code)
    name = LEAGUE_NAMES.get(code, '')[:8]
    
    if 'acc' in r:
        acc = r['acc']
        n = r.get('n', '-')
        delta = acc - v4_acc if v4_acc else None
        delta_str = f"{delta:+.3f}" if delta is not None else "-"
        weak = " ⚠️" if code in WEAK_LEAGUES else ""
        v4_str = f"{v4_acc:.1%}" if v4_acc else "-"
        print(f"{code:<6} {name:<8} {acc:.1%}       {v4_str:<10} {delta_str:<8} {n:<6} ✅{weak}")
        if v4_acc:
            total_imp += delta
            count += 1
    else:
        v4_str2 = f"{v4_acc:.1%}" if v4_acc else "-"
        print(f"{code:<6} {name:<8} {'-':<10} {v4_str2:<10} {'-':<8} {'-':<6} {r.get('status','?')}")

if count > 0:
    avg_imp = total_imp / count
    print(f"\n平均提升: {avg_imp:+.3f} ({avg_imp*100:+.1f}%) over {count} leagues")

summary = {code: {'league_name': LEAGUE_NAMES.get(code, code), **r}
           for code, r in results.items()}
with open(OUT_DIR / 'ah_1year_v5_summary.json', 'w') as f:
    json.dump(summary, f, indent=2)
print(f"\n模型保存目录: {OUT_DIR}")
print("=" * 60)
