#!/usr/bin/env python3
"""
防过拟合版大小球预测训练
- Walk-Forward 验证（模拟真实预测场景）
- 特征选择（去除冗余特征）
- 复杂度受控的模型
- 赔率约束 + 预锁阈值
- 不在测试数据上优化阈值
"""
import json, os, sys, warnings
from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score, log_loss

warnings.filterwarnings('ignore')

# ========== 配置 ==========
REPO_ROOT = Path(__file__).resolve().parents[1]
HISTORICAL_DIR = REPO_ROOT / 'historical_data'
OUTPUT_DIR = REPO_ROOT / 'experiments' / 'goal_number_antioverfit'
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

LEAGUES = {
    'd1': {'name': '德甲',  'has_odds': True},
    'sp1': {'name': '西甲',  'has_odds': True},
    'i1':  {'name': '意甲',  'has_odds': True},
    'e0':  {'name': '英超',  'has_odds': False},
    'f2':  {'name': '法乙',  'has_odds': False},
    'r1':  {'name': '俄超',  'has_odds': False},
    'g1':  {'name': '希腊超','has_odds': False},
    'f1':  {'name': '法甲',  'has_odds': False},
    'e2':  {'name': '英冠',  'has_odds': False},
}

# Walk-Forward 参数
WF_TRAIN_SIZE = 400   # 每次用多少场训练
WF_TEST_SIZE  = 50    # 每次预测多少场
WF_STEP       = 25    # 滚动步长

# 预锁阈值（不在测试集上搜索）
LOCKED_THRESHOLD = 0.63   # 高于63%才下注
MIN_BET_COUNT    = 15     # 窗口内至少15场才下注
MIN_EDGE         = 0.05   # 模型概率 vs 赔率隐含概率，至少5%优势才下注

# 特征上限（防止特征爆炸）
MAX_FEATURES = 25


# ========== 数据加载 ==========
def load_league(code: str) -> pd.DataFrame | None:
    path = HISTORICAL_DIR / f'{code}_history.csv'
    if not path.exists():
        return None
    df = pd.read_csv(path, low_memory=False)
    df['Date'] = pd.to_datetime(df['Date'], errors='coerce')
    df = df.dropna(subset=['Date', 'FTHG', 'FTAG'])
    df = df.sort_values('Date').reset_index(drop=True)
    df['tot'] = pd.to_numeric(df['FTHG'], errors='coerce') + pd.to_numeric(df['FTAG'], errors='coerce')
    df = df.dropna(subset=['tot'])
    df['is_over'] = (df['tot'] > 2.5).astype(int)
    # 赔率列
    ou_over_c = ou_und_c = None
    for oo, uu in [('B365>2.5','B365<2.5'), ('Avg>2.5','Avg<2.5'), ('Max>2.5','Max<2.5')]:
        if oo in df.columns:
            ou_over_c, ou_und_c = oo, uu
            break
    if ou_over_c:
        df[ou_over_c] = pd.to_numeric(df[ou_over_c], errors='coerce')
        df[ou_und_c] = pd.to_numeric(df[ou_und_c], errors='coerce')
        df['over_prob'] = df[ou_over_c].apply(lambda x: 1/x if pd.notna(x) and x > 1 else np.nan)
        df['und_prob']  = df[ou_und_c].apply(lambda x: 1/x if pd.notna(x) and x > 1 else np.nan)
        df['market_tot'] = (1 / (df['over_prob'] + df['und_prob'])) * df['over_prob']  # 隐含大球概率
    return df


# ========== 特征工程（防泄漏） ==========
def build_features(df: pd.DataFrame, has_odds: bool) -> pd.DataFrame:
    """
    用 expanding window（累计到当前）计算球队统计特征，
    保证没有任何"未来信息泄漏"。
    """
    th = defaultdict(lambda: {
        'gf': [], 'ga': [], 'tot': [], 'over': [],
        'home_gf': [], 'home_ga': [], 'home_over': [],
        'away_gf': [], 'away_ga': [], 'away_over': [],
        'gf_3': [], 'ga_3': [], 'gf_5': [], 'ga_5': [],
    })

    rows = []
    for _, row in df.iterrows():
        home = str(row.get('HomeTeam', ''))
        away = str(row.get('AwayTeam', ''))
        hk = (home, 'home')
        ak = (away, 'away')

        h = th[hk]
        a = th[ak]

        def avg(v, d=1.5):
            return float(np.mean(v)) if v else d

        def last_n(v, n, d=1.5):
            v = list(v)
            return float(np.mean(v[-n:])) if len(v) >= n else (float(np.mean(v)) if v else d)

        def over_streak(v, val=1):
            s = 0
            for x in reversed(v):
                if x == val: s += 1
                else: break
            return s

        r = {
            # --- 主队统计 ---
            'home_gf_avg':   avg(h['gf']),
            'home_ga_avg':   avg(h['ga']),
            'home_tot_avg':  avg(h['tot'], 2.6),
            'home_over_rate':avg(h['over'], 0.52),
            'home_gf_3':    last_n(h['gf_3'], 3),
            'home_gf_5':    last_n(h['gf_5'], 5),
            'home_ga_3':    last_n(h['ga_3'], 3),
            'home_ga_5':    last_n(h['ga_5'], 5),
            'home_over_3':  last_n(h['over'], 3, 0.52),
            'home_over_5':  last_n(h['over'], 5, 0.52),
            'home_hgf_avg': avg(h['home_gf'], 1.5),   # 主场进球
            'home_hga_avg': avg(h['home_ga'], 1.3),
            'home_ovrstk':  over_streak(h['over'], 1),
            'home_undrstk': over_streak(h['over'], 0),

            # --- 客队统计 ---
            'away_gf_avg':   avg(a['gf']),
            'away_ga_avg':   avg(a['ga']),
            'away_tot_avg':  avg(a['tot'], 2.6),
            'away_over_rate':avg(a['over'], 0.52),
            'away_gf_3':    last_n(a['gf_3'], 3),
            'away_gf_5':    last_n(a['gf_5'], 5),
            'away_ga_3':    last_n(a['ga_3'], 3),
            'away_ga_5':    last_n(a['ga_5'], 5),
            'away_over_3':  last_n(a['over'], 3, 0.52),
            'away_over_5':  last_n(a['over'], 5, 0.52),
            'away_agf_avg': avg(a['away_gf'], 1.2),   # 客场进球
            'away_aga_avg': avg(a['away_ga'], 1.4),
            'away_ovrstk':  over_streak(a['over'], 1),
            'away_undrstk': over_streak(a['over'], 0),

            # --- 衍生特征 ---
            'exp_g':   (avg(h['gf']) + avg(a['ga'])) / 2,
            'exp_tot': (avg(h['tot'], 2.6) + avg(a['tot'], 2.6)) / 2,
            'over_diff': avg(h['over'], 0.52) - avg(a['over'], 0.52),
            'tot_diff':  avg(h['tot'], 2.6) - avg(a['tot'], 2.6),
            'h2h_over': 0.52,   # 历史默认值，后续可改进
            'form_diff': avg(h['over'], 0.52) - avg(a['over'], 0.52),

            # --- 赔率特征（如果有） ---
            'is_over': row['is_over'],
        }

        if has_odds:
            r['mkt_over_prob'] = row.get('market_tot', np.nan)
            r['mkt_over'] = row.get('over_prob', np.nan)
            r['mkt_und']  = row.get('und_prob', np.nan)
            r['mkt_margin'] = (row.get('over_prob', 0) + row.get('und_prob', 0) - 1
                               if pd.notna(row.get('over_prob')) and pd.notna(row.get('und_prob'))
                               else np.nan)

        rows.append(r)

        # === 更新统计（expanding，到当前为止）===
        tot = float(row['tot'])
        io  = int(row['is_over'])
        hg  = float(row['FTHG'])
        ag  = float(row['FTAG'])

        h['gf'].append(tot); h['ga'].append(tot)
        h['tot'].append(tot); h['over'].append(io)
        h['gf_3'].append(tot); h['ga_3'].append(tot)
        h['gf_5'].append(tot); h['ga_5'].append(tot)

        a['gf'].append(tot); a['ga'].append(tot)
        a['tot'].append(tot); a['over'].append(io)
        a['gf_3'].append(tot); a['ga_3'].append(tot)
        a['gf_5'].append(tot); a['ga_5'].append(tot)

        if pd.notna(hg):
            h['home_gf'].append(hg); h['home_ga'].append(ag); h['home_over'].append(io)
        if pd.notna(ag):
            a['away_gf'].append(ag); a['away_ga'].append(hg); a['away_over'].append(io)

    return pd.DataFrame(rows)


# ========== 特征选择（基于互信息） ==========
def select_features(X: pd.DataFrame, y: pd.Series, max_features: int = MAX_FEATURES) -> list:
    """用互信息选出最重要的特征，剔除冗余特征。"""
    from sklearn.feature_selection import mutual_info_classif

    X_filled = X.fillna(X.median())
    mi_scores = mutual_info_classif(X_filled, y, discrete_features=False, random_state=42)
    feat_mi = sorted(zip(X.columns, mi_scores), key=lambda x: -x[1])

    # 贪心剔除高相关特征
    selected = []
    for feat, score in feat_mi:
        if len(selected) >= max_features:
            break
        # 避免与已选特征高度相关
        if selected:
            corr_with_selected = X_filled[selected].corrwith(X_filled[feat]).abs().max()
            if corr_with_selected < 0.90:  # 相关系数<0.9才保留
                selected.append(feat)
        else:
            selected.append(feat)
    return selected


# ========== Walk-Forward 验证 ==========
def walk_forward_validate(X, y, model_fn, threshold, has_odds=False, min_bets=MIN_BET_COUNT):
    """
    滚动窗口验证：
    用过去 WF_TRAIN_SIZE 场训练，预测接下来 WF_TEST_SIZE 场，
    每隔 WF_STEP 场滚动一次。
    返回：整体准确率、 bet准确率、总bet数量
    """
    n = len(X)
    if n < WF_TRAIN_SIZE + WF_TEST_SIZE:
        return None

    all_probs = []
    all_preds = []
    all_true  = []
    all_bet_mask = []

    n_windows = 0

    for start in range(0, n - WF_TRAIN_SIZE - WF_TEST_SIZE + 1, WF_STEP):
        train_end = start + WF_TRAIN_SIZE
        test_end  = train_end + WF_TEST_SIZE

        X_tr, X_te = X.iloc[start:train_end], X.iloc[train_end:test_end]
        y_tr, y_te = y.iloc[start:train_end], y.iloc[train_end:test_end]

        if len(X_tr) < 100 or len(X_te) < 10:
            continue

        # 标准化
        scaler = StandardScaler()
        X_tr_s = scaler.fit_transform(X_tr.fillna(X_tr.median()))
        X_te_s = scaler.transform(X_te.fillna(X_tr.median()))

        model = model_fn()
        model.fit(X_tr_s, y_tr)

        probs = model.predict_proba(X_te_s)[:, 1]
        preds = (probs >= threshold).astype(int)

        all_probs.extend(probs)
        all_preds.extend(preds)
        all_true.extend(y_te.values)
        all_bet_mask.extend((probs >= threshold) | (probs <= (1 - threshold)))

        n_windows += 1

    if n_windows == 0:
        return None

    all_probs = np.array(all_probs)
    all_preds = np.array(all_preds)
    all_true  = np.array(all_true)
    all_bet_mask = np.array(all_bet_mask)

    # 整体准确率
    overall_acc = accuracy_score(all_true, all_preds)

    # 只在"下注"样本上计算准确率
    bet_probs  = all_probs[all_bet_mask]
    bet_preds  = all_preds[all_bet_mask]
    bet_true   = all_true[all_bet_mask]

    if len(bet_true) < min_bets:
        bet_acc = overall_acc
        bet_count = len(bet_true)
    else:
        bet_acc = accuracy_score(bet_true, bet_preds)
        bet_count = len(bet_true)

    return {
        'overall_acc': overall_acc,
        'bet_acc':     bet_acc,
        'bet_count':   bet_count,
        'n_windows':   n_windows,
    }


# ========== 训练单个联赛 ==========
def train_league(code: str):
    print(f"\n{'='*60}")
    print(f"  {LEAGUES[code]['name']} ({code}) - 防过拟合版")
    print(f"{'='*60}")

    df = load_league(code)
    if df is None:
        print("  [SKIP] 无数据")
        return None

    has_odds = LEAGUES[code]['has_odds']
    print(f"  数据: {len(df)} 场, 赔率: {'有' if has_odds else '无'}")
    print(f"  大球率: {df['is_over'].mean():.1%}")

    # 至少要1000场才有意义
    if len(df) < 1000:
        print("  [SKIP] 数据不足1000场")
        return None

    # 特征构建（expanding window，无泄漏）
    print("  构建特征...")
    feat_df = build_features(df, has_odds)

    target = 'is_over'
    all_cols = [c for c in feat_df.columns if c != target]

    # 去掉NaN>50%的列
    nan_ratio = feat_df[all_cols].isna().mean()
    valid_cols = [c for c in all_cols if nan_ratio[c] < 0.5]
    X_all = feat_df[valid_cols].fillna(0)
    y_all = feat_df[target].astype(int)

    print(f"  原始特征: {len(valid_cols)}")

    # 特征选择（用前半段数据）
    split_idx = int(len(X_all) * 0.5)
    selected_feats = select_features(X_all.iloc[:split_idx], y_all.iloc[:split_idx])
    print(f"  精选特征: {len(selected_feats)} → {selected_feats[:8]}...")

    X_sel = X_all[selected_feats]

    # ===== 模型候选（低复杂度）=====
    def rf_model():
        return RandomForestClassifier(
            n_estimators=150,
            max_depth=7,          # 限制深度
            min_samples_leaf=25,  # 强制叶子有足够样本 ← 关键防过拟合
            max_features=0.6,
            random_state=42,
            n_jobs=-1,
        )

    def gb_model():
        return GradientBoostingClassifier(
            n_estimators=120,
            max_depth=4,          # 浅树
            learning_rate=0.05,    # 低学习率
            min_samples_leaf=30,
            subsample=0.8,
            random_state=42,
        )

    def lr_model():
        return LogisticRegression(
            C=0.1,               # 强L2正则
            max_iter=1000,
            random_state=42,
        )

    models = [
        ('RF_constrained', rf_model),
        ('GB_constrained', gb_model),
        ('LR_regularized', lr_model),
    ]

    # ===== Walk-Forward 验证 =====
    print(f"\n  Walk-Forward 验证 (train={WF_TRAIN_SIZE}, test={WF_TEST_SIZE}, step={WF_STEP})")
    print(f"  预锁阈值: {LOCKED_THRESHOLD}")

    best_result = None
    best_model_name = None
    best_model_fn = None

    for name, fn in models:
        result = walk_forward_validate(X_sel, y_all, fn, LOCKED_THRESHOLD, has_odds)
        if result:
            print(f"  {name}: "
                  f"整体={result['overall_acc']:.1%}, "
                  f"投注={result['bet_acc']:.1%} (n={result['bet_count']}), "
                  f"窗口={result['n_windows']}")
            if best_result is None or result['bet_acc'] > best_result['bet_acc']:
                best_result = result
                best_model_name = name
                best_model_fn = fn

    if best_result is None:
        print("  [FAIL] 所有模型验证失败")
        return None

    print(f"\n  最佳模型: {best_model_name}, 投注准确率: {best_result['bet_acc']:.1%}")

    # ===== 用全部数据训练最终模型 =====
    print("  训练最终模型（全量数据）...")
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X_sel.fillna(X_sel.median()))

    final_model = best_model_fn()
    final_model.fit(X_scaled, y_all)

    # ===== 保存 =====
    out_dir = OUTPUT_DIR / code
    out_dir.mkdir(parents=True, exist_ok=True)

    model_data = {
        'model':       final_model,
        'scaler':      scaler,
        'features':    selected_feats,
        'threshold':   LOCKED_THRESHOLD,
        'league':      code,
        'league_name': LEAGUES[code]['name'],
        'wf_result':   {k: float(v) if isinstance(v, (np.floating, float)) else v
                        for k, v in best_result.items()},
        'has_odds':    has_odds,
    }
    joblib.dump(model_data, out_dir / 'model.pkl')

    config = {
        'league':      code,
        'league_name':LEAGUES[code]['name'],
        'threshold':  LOCKED_THRESHOLD,
        'features':    selected_feats,
        'feature_count': len(selected_feats),
        'model_name':  best_model_name,
        'wf_overall_acc': float(best_result['overall_acc']),
        'wf_bet_acc':  float(best_result['bet_acc']),
        'wf_bet_count': int(best_result['bet_count']),
        'wf_n_windows': int(best_result['n_windows']),
        'has_odds':    has_odds,
    }
    with open(out_dir / 'config.json', 'w', encoding='utf-8') as f:
        json.dump(config, f, ensure_ascii=False, indent=2)

    print(f"  保存至: {out_dir}/")

    status = "✓ 达标" if best_result['bet_acc'] >= 0.70 else "✗ 未达70%"
    print(f"\n  {status} (投注准确率 {best_result['bet_acc']:.1%})")
    return config


# ========== 主函数 ==========
def main():
    if len(sys.argv) < 2:
        print("用法: python3 train_anti_overfit.py <联赛code>")
        print("可用:", ", ".join(LEAGUES.keys()))
        sys.exit(1)

    code = sys.argv[1].lower()
    if code not in LEAGUES:
        print(f"未知联赛: {code}")
        sys.exit(1)

    result = train_league(code)
    if result:
        print(f"\n{'='*50}")
        print(f"最终结果: {result['league_name']}")
        print(f"  Walk-Forward 投注准确率: {result['wf_bet_acc']:.1%}")
        print(f"  投注场次: {result['wf_bet_count']}")
        print(f"  特征数: {result['feature_count']}")
        print(f"  模型: {result['model_name']}")
        print(f"  阈值: {result['threshold']}")


if __name__ == '__main__':
    main()
