#!/usr/bin/env python3
"""亚盘模型训练 V3 — 防过拟合版
时间序列分割：train=2025.1-2025.8, val=2025.9, test=2025.10-2025.12
"""
import json, warnings, sys
from pathlib import Path
from collections import defaultdict
import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression, RidgeClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score
import joblib

warnings.filterwarnings('ignore')
np.random.seed(42)

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
HIST = REPO / 'historical_data'
ODDS_DIR = REPO / 'artifacts' / 'odds_api_history'
OUT = REPO / 'models' / 'ah_1year_v3'
OUT.mkdir(parents=True, exist_ok=True)

ODDS_AH = ['B365AHH', 'B365AHA', 'AvgAHH', 'AvgAHA', 'PAHH', 'PAHA', 'AHh']
ODDS_OU = ['B365>2.5', 'B365<2.5', 'Avg>2.5', 'Avg<2.5']

LEAGUE_NAMES = {
    'e0':'英超','e1':'英甲','e2':'英冠','e2n':'英乙','e3':'英格兰第五级',
    'd1':'德甲','d2':'德乙','f1':'法甲','f2':'法乙',
    'i1':'意甲','i2':'意乙','sp1':'西甲','sp2':'西乙',
    'tr1':'土超','p1':'葡超','b1':'比甲','g1':'希腊超',
    'n1':'荷兰甲','a1':'澳超','r1':'俄超','br1':'巴甲','m1':'美职联',
    'sc1':'苏超','cz1':'捷甲',
}


def make_hist():
    return defaultdict(lambda: {
        'gf':[],'ga':[],'tot':[],'win':[],'und':[],
        'hgf':[],'hga':[],'hgc':[],'agf':[],'aga':[],'agc':[],'mom':[],'rf':[]
    })


def build_features(df, team_hist):
    """构建特征，返回 DataFrame；同步更新 team_hist（比赛结果追加到历史）"""
    rows = []
    for _, m in df.iterrows():
        h, a = str(m['HomeTeam']), str(m['AwayTeam'])
        hh, aa = team_hist[h], team_hist[a]
        fthg = float(m.get('FTHG', 0) or 0)
        ftag = float(m.get('FTAG', 0) or 0)

        def mn(v, n):
            v = list(v)
            return float(np.mean(v[-n:])) if len(v) >= n else (float(np.mean(v)) if v else 0.0)
        def std(v):
            v = list(v)
            return float(np.std(v[-10:])) if len(v) >= 2 else 0.5

        d = {}
        for p, s in [('home', hh), ('away', aa)]:
            d[f'{p}_gavg'] = np.mean(s['gf']) if s['gf'] else 1.5
            d[f'{p}_gavga'] = np.mean(s['ga']) if s['ga'] else 1.5
            d[f'{p}_tavg'] = np.mean(s['tot']) if s['tot'] else 2.5
            d[f'{p}_winr'] = np.mean(s['win']) if s['win'] else 0.5
            d[f'{p}_undr'] = np.mean(s['und']) if s['und'] else 0.5
            d[f'{p}_gstd'] = std(s.get('gf',[]))
            d[f'{p}_hgf'] = np.mean(s['hgf']) if s['hgf'] else d[f'{p}_gavg']
            d[f'{p}_hga'] = np.mean(s['hga']) if s['hga'] else d[f'{p}_gavga']
            d[f'{p}_hgc'] = np.mean(s['hgc']) if s['hgc'] else 0.3
            d[f'{p}_agf'] = np.mean(s['agf']) if s['agf'] else d[f'{p}_gavg']
            d[f'{p}_aga'] = np.mean(s['aga']) if s['aga'] else d[f'{p}_gavga']
            d[f'{p}_agc'] = np.mean(s['agc']) if s['agc'] else 0.3
            d[f'{p}_mom'] = np.mean(s['mom'][-5:]) if len(s.get('mom',[])) >= 2 else 0.0
            d[f'{p}_rf'] = np.mean(s['rf'][-5:]) if len(s.get('rf',[])) >= 2 else 0.0
            for n in [3, 5, 10]:
                d[f'{p}_gf{n}'] = mn(s.get('gf',[]), n)
                d[f'{p}_ga{n}'] = mn(s.get('ga',[]), n)

        d['hg_diff'] = d['home_gavg'] - d['away_agf']
        d['ga_diff'] = d['home_gavga'] - d['away_aga']
        d['tot_diff'] = d['home_tavg'] - d['away_tavg']
        d['win_diff'] = d['home_winr'] - d['away_winr']
        d['mom_diff'] = d['home_mom'] - d['away_mom']

        for c in ODDS_AH + ODDS_OU:
            v = m.get(c)
            if c == 'AHh':
                d[c] = float(v) if pd.notna(v) else 0.0
            else:
                d[c] = float(v) if pd.notna(v) and float(v) > 1 else 0.0

        b365h = d.get('B365AHH', 0)
        b365a = d.get('B365AHA', 0)
        if b365h > 1 and b365a > 1:
            d['ah_ratio'] = b365h / b365a
            d['ah_prob_home'] = b365a / (b365h + b365a)
        else:
            d['ah_ratio'] = 1.0
            d['ah_prob_home'] = 0.5

        rows.append(d)

        # 追加比赛结果到历史（用于下一场比赛的特征）
        for arr in [hh, aa]:
            arr['gf'].append(fthg); arr['ga'].append(ftag); arr['tot'].append(fthg + ftag)
            arr['win'].append(1 if fthg > ftag else 0); arr['und'].append(1 if fthg < ftag else 0)
        hh['hgf'].append(fthg); hh['hga'].append(ftag); hh['hgc'].append(1 if ftag == 0 else 0)
        aa['agf'].append(ftag); aa['aga'].append(fthg); aa['agc'].append(1 if fthg == 0 else 0)
        for arr in [hh, aa]:
            g = arr['gf']
            if len(g) >= 2:
                arr['mom'].append(np.mean(g[-3:]) - np.mean(g[-6:-3]) if len(g) >= 6 else 0)
                arr['rf'].append(np.mean(arr['win'][-3:]) - np.mean(arr['win'][:-3]) if len(arr['win']) >= 6 else 0)

    return pd.DataFrame(rows)


def get_ah_outcome(row):
    fthg = float(row.get('FTHG', 0) or 0)
    ftag = float(row.get('FTAG', 0) or 0)
    ahh = float(row.get('AHh', 0) or 0)
    adj = fthg - ahh
    return 1 if adj > ftag else (0 if adj < ftag else -1)  # 1=H wins AH, 0=A wins AH, -1=push


def run_league(code):
    # 优先读赔率合并文件（有B365数据），没有则读历史文件
    odds_file = ODDS_DIR / f"{code}_odds_api_merged.csv"
    hist_file = HIST / f"{code}_history.csv"

    dfs = []
    if odds_file.exists():
        try:
            od = pd.read_csv(odds_file, low_memory=False, encoding='utf-8-sig')
            od['_source'] = 'odds'
            dfs.append(od)
        except: pass
    if hist_file.exists():
        try:
            hd = pd.read_csv(hist_file, low_memory=False, encoding='utf-8-sig')
            hd['_source'] = 'hist'
            dfs.append(hd)
        except: pass

    if not dfs:
        return None

    df = pd.concat(dfs, ignore_index=True)
    df = df.drop_duplicates(subset=['Date', 'HomeTeam', 'AwayTeam'], keep='first')
    df['Date'] = pd.to_datetime(df['Date'], dayfirst=True, errors='coerce')
    df = df[df['Date'].notna()].sort_values('Date').reset_index(drop=True)
    df['FTHG'] = pd.to_numeric(df['FTHG'], errors='coerce')
    df['FTAG'] = pd.to_numeric(df['FTAG'], errors='coerce')
    df['AHh'] = pd.to_numeric(df.get('AHh', 0), errors='coerce').fillna(0)
    df['B365AHH'] = pd.to_numeric(df.get('B365AHH', np.nan), errors='coerce')
    df['B365AHA'] = pd.to_numeric(df.get('B365AHA', np.nan), errors='coerce')
    df = df[df['FTHG'].notna() & df['FTAG'].notna()].reset_index(drop=True)

    has_ah = df['B365AHH'].notna() & (df['B365AHH'] > 1.0)

    # 2025年内数据
    y2025 = df[df['Date'].dt.year == 2025].copy()
    train_2025 = y2025[y2025['Date'].dt.month <= 8]  # 1-8月训练
    val_2025 = y2025[y2025['Date'].dt.month == 9]    # 9月验证
    test_2025 = y2025[y2025['Date'].dt.month >= 10]  # 10-12月测试

    # 从全部历史中取1-8月训练（2025）
    train_df = train_2025[train_2025.apply(lambda r: has_ah.get(r.name, False), axis=1)].reset_index(drop=True)
    val_df = val_2025[val_2025.apply(lambda r: has_ah.get(r.name, False), axis=1)].reset_index(drop=True)
    test_df = test_2025[test_2025.apply(lambda r: has_ah.get(r.name, False), axis=1)].reset_index(drop=True)

    # 也加入2024年10-12月数据作为训练（增加训练量）
    train_2024 = df[(df['Date'].dt.year == 2024) & (df['Date'].dt.month >= 10)]
    train_2024 = train_2024[train_2024.apply(lambda r: has_ah.get(r.name, False), axis=1)].reset_index(drop=True)

    if len(train_df) < 30 or len(val_df) < 5 or len(test_df) < 5:
        return None

    print(f"  train={len(train_df)} val={len(val_df)} test={len(test_df)}", end=" ")

    # 构建特征
    th_train = make_hist()
    X_train = build_features(train_df, th_train).fillna(0).replace([np.inf, -np.inf], 0)
    y_train = np.array([get_ah_outcome(r) for _, r in train_df.iterrows()])
    y_train = np.where(y_train == -1, 1, y_train)  # push→按主队算（不计入）

    th_val = make_hist()
    X_val = build_features(val_df, th_val).fillna(0).replace([np.inf, -np.inf], 0)
    y_val = np.array([get_ah_outcome(r) for _, r in val_df.iterrows()])
    y_val = np.where(y_val == -1, 1, y_val)

    th_test = make_hist()
    X_test = build_features(test_df, th_test).fillna(0).replace([np.inf, -np.inf], 0)
    y_test = np.array([get_ah_outcome(r) for _, r in test_df.iterrows()])
    y_test = np.where(y_test == -1, 1, y_test)

    best_val_acc = 0
    best_config = None

    # 适度正则化模型（避免复杂模型）
    models = [
        ('LR_C0.01', LogisticRegression(max_iter=1000, C=0.01, random_state=42)),
        ('LR_C0.1', LogisticRegression(max_iter=1000, C=0.1, random_state=42)),
        ('LR_C1', LogisticRegression(max_iter=1000, C=1.0, random_state=42)),
        ('RF_d3', RandomForestClassifier(n_estimators=100, max_depth=3, random_state=42, min_samples_leaf=10)),
        ('RF_d4', RandomForestClassifier(n_estimators=100, max_depth=4, random_state=42, min_samples_leaf=8)),
        ('ET_d3', ExtraTreesClassifier(n_estimators=100, max_depth=3, random_state=42, min_samples_leaf=10)),
        ('GB_d2', GradientBoostingClassifier(n_estimators=50, max_depth=2, random_state=42, learning_rate=0.05)),
        ('GB_d3', GradientBoostingClassifier(n_estimators=100, max_depth=3, random_state=42, learning_rate=0.05)),
    ]

    for sname, scaler in [('std', StandardScaler()), ('rob', RobustScaler())]:
        for mname, clf in models:
            try:
                pipe = Pipeline([
                    ('scaler', scaler),
                    ('imputer', SimpleImputer(fill_value=0)),
                    ('clf', clf)
                ])
                pipe.fit(X_train.values, y_train)
                proba = pipe.predict_proba(X_val.values)[:, 1]

                # 验证集选阈值（只用验证集）
                for th_p in np.arange(0.50, 0.65, 0.01):
                    for direction in ['over', 'under']:
                        if direction == 'over':
                            mask = (proba >= th_p)
                            bet_y = np.ones(len(y_val), dtype=int)
                        else:
                            mask = (proba <= (1 - th_p))
                            bet_y = np.zeros(len(y_val), dtype=int)

                        if mask.sum() < 5:
                            continue

                        preds = bet_y.copy()
                        preds[~mask] = -1  # 不投注
                        valid_mask = preds != -1
                        if valid_mask.sum() < 5:
                            continue

                        val_acc = accuracy_score(y_val[valid_mask], preds[valid_mask])
                        if val_acc > best_val_acc:
                            best_val_acc = val_acc
                            best_config = {
                                'model_name': f'{mname}_{sname}',
                                'pipe': pipe,
                                'threshold': th_p,
                                'direction': direction,
                                'val_bets': int(valid_mask.sum()),
                                'val_acc': float(val_acc),
                                'val_expected_roi': float(val_acc - 0.5)  # 粗估ROI
                            }
            except Exception as e:
                pass

    if best_config is None:
        return None

    cfg = best_config
    pipe = cfg['pipe']

    # 在测试集上评估（最终结果）
    test_proba = pipe.predict_proba(X_test.values)[:, 1]
    th_p = cfg['threshold']
    direction = cfg['direction']

    if direction == 'over':
        mask = (test_proba >= th_p)
        test_preds = np.ones(len(y_test), dtype=int)
    else:
        mask = (test_proba <= (1 - th_p))
        test_preds = np.zeros(len(y_test), dtype=int)

    test_preds[~mask] = -1
    valid_test = test_preds != -1
    test_acc = accuracy_score(y_test[valid_test], test_preds[valid_test]) if valid_test.sum() > 0 else 0

    # 计算 bet_count（可投注场次）
    test_bet_count = int(valid_test.sum())
    test_correct = int((y_test[valid_test] == test_preds[valid_test]).sum())

    print(f"→ val_acc={cfg['val_acc']:.1%} test_acc={test_acc:.1%}({test_correct}/{test_bet_count})", end=" ")

    # 保存
    league_out = OUT / code
    league_out.mkdir(parents=True, exist_ok=True)
    joblib.dump(pipe, league_out / 'optimized_pipeline.pkl')

    save_cfg = {k: v for k, v in cfg.items() if k != 'pipe'}
    save_cfg['league'] = code
    save_cfg['league_name'] = LEAGUE_NAMES.get(code, code)
    save_cfg['train_count'] = len(train_df)
    save_cfg['val_count'] = len(val_df)
    save_cfg['test_count'] = len(test_df)
    save_cfg['test_acc'] = float(test_acc)
    save_cfg['test_bets'] = test_bet_count
    save_cfg['test_correct'] = test_correct

    with open(league_out / 'optimized_config.json', 'w') as f:
        json.dump(save_cfg, f, ensure_ascii=False, indent=2)

    print(f"✅" if test_acc >= 0.52 else "❌", end="")
    return save_cfg


print("=== 亚盘模型训练 V3 — 防过拟合 ===")
print("时间分割: train=2025.1-8, val=2025.9, test=2025.10-12\n")

TARGET = ['e0','e1','e2','e2n','e3','d1','d2','f1','f2','i1','i2','sp1','sp2',
          'tr1','p1','b1','g1','n1','a1','r1','br1','m1','sc1','cz1']

results = []
for code in TARGET:
    sys.stdout.flush()
    code_name = LEAGUE_NAMES.get(code, code)
    print(f"\n{code_name}({code}):", end=" ", flush=True)
    r = run_league(code)
    if r:
        results.append(r)

print("\n\n=== 汇总（按测试集准确率排序）===")
print(f"{'联赛':<8} {'val_acc':>8} {'test_acc':>8} {'test场次':>8} {'val场次':>8} {'模型'}")
print("-" * 65)
for r in sorted(results, key=lambda x: -x['val_acc']):
    flag = '✅' if r['test_acc'] >= 0.52 else '❌'
    name = f"{r['league_name']}({r['league']})"
    print(f"{name:<12} {r['val_acc']:>7.1%} {r['test_acc']:>7.1%} {flag} {r['test_bets']:>5}场 {r['val_bets']:>5}场 {r['model_name']}")
test_avg = np.mean([r['test_acc'] for r in results]) if results else 0
over52 = sum(1 for r in results if r['test_acc'] >= 0.52)
print(f"\n{len(results)}个联赛 | 测试集52%+准确率: {over52}个 | 平均准确率: {test_avg:.1%}")

if results:
    with open(OUT / 'ah_summary.json', 'w') as f:
        json.dump(results, f, ensure_ascii=False, indent=2)
    print(f"结果已保存到 {OUT}")