#!/usr/bin/env python3
"""亚盘让球盘模型训练 — 1year-V2
使用真实赔率数据：Odds API merge文件(2025.1.1-2026.2.28)
专注达到75%+投注准确率"""
import json, warnings, sys
from pathlib import Path
from collections import defaultdict
import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier, AdaBoostClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score
from sklearn.model_selection import TimeSeriesSplit
import joblib

warnings.filterwarnings('ignore')
np.random.seed(42)

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
HIST = REPO / 'historical_data'
ODIR = REPO / 'artifacts' / 'odds_api_history'
OUT = REPO / 'models' / 'ah_1year_v2'
OUT.mkdir(parents=True, exist_ok=True)

ODDS_AH = ['B365AHH', 'B365AHA', 'AvgAHH', 'AvgAHA', 'PAHH', 'PAHA', 'AHh']
ODDS_OU = ['B365>2.5', 'B365<2.5', 'Avg>2.5', 'Avg<2.5']

LEAGUE_NAMES = {
    'd1':'德甲','d2':'德乙','f1':'法甲','f2':'法乙','cz1':'捷甲',
    'sc1':'苏超','e1':'英甲','e2':'英冠','e2n':'英乙','e3':'英格兰第五级',
    'i1':'意甲','sp1':'西甲','tr1':'土超','p1':'葡超',
    'e0':'英超','sp2':'西乙','i2':'意乙','b1':'比甲','g1':'希腊超','n1':'荷兰甲',
    'a1':'澳超','r1':'俄超','br1':'巴甲','m1':'美职联'
}

# 联赛 → 使用merged文件（有真实OddsAPI赔率）
MERGED_LEAGUES = {'a1', 'r1', 'br1', 'm1'}

# 目标训练日期范围
DATE_START = '2025-01-01'
DATE_END   = '2026-02-28'


def make_hist():
    return defaultdict(lambda: {
        'gf':[],'ga':[],'tot':[],'win':[],'und':[],
        'gf3':[],'ga3':[],'gf5':[],'ga5':[],'gf10':[],'ga10':[],
        'hgf':[],'hga':[],'hgc':[],'agf':[],'aga':[],'agc':[],'mom':[],'rf':[]
    })


def build_features(df, team_hist):
    rows = []
    for _, m in df.iterrows():
        h, a = str(m['HomeTeam']), str(m['AwayTeam'])
        hh, aa = team_hist[h], team_hist[a]
        fthg = float(m.get('FTHG', 0) or 0)
        ftag = float(m.get('FTAG', 0) or 0)

        def mn(v, n):
            v = list(v)
            return float(np.mean(v[-n:])) if len(v) >= n else (float(np.mean(v)) if v else 0.0)
        def std(v):
            v = list(v)
            return float(np.std(v[-10:])) if len(v) >= 2 else 0.5

        d = {}
        for p, s in [('home', hh), ('away', aa)]:
            d[f'{p}_gavg']  = np.mean(s['gf']) if s['gf'] else 1.5
            d[f'{p}_gavga'] = np.mean(s['ga']) if s['ga'] else 1.5
            d[f'{p}_tavg']  = np.mean(s['tot']) if s['tot'] else 2.5
            d[f'{p}_winr']  = np.mean(s['win']) if s['win'] else 0.5
            d[f'{p}_undr']  = np.mean(s['und']) if s['und'] else 0.5
            d[f'{p}_gstd']  = std(s.get('gf',[]))
            d[f'{p}_hgf']   = np.mean(s['hgf']) if s['hgf'] else d[f'{p}_gavg']
            d[f'{p}_hga']   = np.mean(s['hga']) if s['hga'] else d[f'{p}_gavga']
            d[f'{p}_hgc']   = np.mean(s['hgc']) if s['hgc'] else 0.3
            d[f'{p}_agf']   = np.mean(s['agf']) if s['agf'] else d[f'{p}_gavg']
            d[f'{p}_aga']   = np.mean(s['aga']) if s['aga'] else d[f'{p}_gavga']
            d[f'{p}_agc']   = np.mean(s['agc']) if s['agc'] else 0.3
            d[f'{p}_mom']   = np.mean(s['mom'][-5:]) if len(s.get('mom',[])) >= 2 else 0.0
            d[f'{p}_rf']    = np.mean(s['rf'][-5:]) if len(s.get('rf',[])) >= 2 else 0.0
            for n in [3, 5, 10]:
                d[f'{p}_gf{n}'] = mn(s.get('gf',[]), n)
                d[f'{p}_ga{n}'] = mn(s.get('ga',[]), n)

        d['hg_diff']  = d['home_gavg'] - d['away_agf']
        d['ga_diff']  = d['home_gavga'] - d['away_aga']
        d['tot_diff'] = d['home_tavg'] - d['away_tavg']
        d['win_diff'] = d['home_winr'] - d['away_winr']
        d['mom_diff'] = d['home_mom'] - d['away_mom']

        for c in ODDS_AH + ODDS_OU:
            v = m.get(c)
            if c == 'AHh':
                d[c] = float(v) if pd.notna(v) else 0.0
            else:
                d[c] = float(v) if pd.notna(v) and float(v) > 1 else 0.0

        b365h = d.get('B365AHH', 0)
        b365a = d.get('B365AHA', 0)
        if b365h > 1 and b365a > 1:
            d['ah_ratio'] = b365h / b365a
            d['ah_prob_home'] = b365a / (b365h + b365a)
        else:
            d['ah_ratio'] = 1.0
            d['ah_prob_home'] = 0.5

        rows.append(d)

        # 更新历史
        for arr in [hh, aa]:
            arr['gf'].append(fthg + ftag); arr['ga'].append(ftag + fthg); arr['tot'].append(fthg + ftag)
            arr['win'].append(1 if fthg > ftag else 0); arr['und'].append(1 if fthg + ftag < 2.5 else 0)
        hh['hgf'].append(fthg); hh['hga'].append(ftag); hh['hgc'].append(1 if ftag == 0 else 0)
        aa['agf'].append(ftag); aa['aga'].append(fthg); aa['agc'].append(1 if fthg == 0 else 0)
        for arr in [hh, aa]:
            g = arr['gf']
            if len(g) >= 2:
                arr['mom'].append(np.mean(g[-3:]) - np.mean(g[-6:-3]) if len(g) >= 6 else 0)
                arr['rf'].append(np.mean(arr['win'][-3:]) - np.mean(arr['win'][:-3]) if len(arr['win']) >= 6 else 0)

    return pd.DataFrame(rows)


def get_ah_outcome(row):
    fthg = float(row.get('FTHG', 0) or 0)
    ftag = float(row.get('FTAG', 0) or 0)
    ahh = float(row.get('AHh', 0) or 0)
    return 1 if (fthg - ftag - ahh) > 0 else 0


def load_league_data(code):
    """加载联赛数据：优先用merged文件（有真实赔率）"""
    if code in MERGED_LEAGUES:
        f = ODIR / f'{code}_odds_api_merged.csv'
        if f.exists():
            df = pd.read_csv(f, low_memory=False, encoding='utf-8-sig')
            # odds_source列表示来自OddsAPI的行
            # 只取有赔率的行
            if 'odds_source' in df.columns:
                df = df[df['odds_source'] == 'odds_api'].copy()
            print(f'    [{code}] 使用merged文件: {len(df)}行(真实赔率)')
            return df

    # 默认从历史文件加载
    f = HIST / f'{code}_history.csv'
    if not f.exists():
        return pd.DataFrame()
    df = pd.read_csv(f, low_memory=False, encoding='utf-8-sig')
    print(f'    [{code}] 使用历史文件: {len(df)}行')
    return df


def parse_date(d):
    if pd.isna(d): return None
    d = str(d).strip()
    import re
    m = re.match(r'^(\d{4})-(\d{2})-(\d{2})$', d)
    if m: return f'{m.group(1)}-{m.group(2)}-{m.group(3)}'
    m = re.match(r'^(\d{2})/(\d{2})/(\d{4})$', d)
    if m: return f'{m.group(3)}-{m.group(2)}-{m.group(1)}'
    return None


def prepare_data(code):
    df = load_league_data(code)
    if len(df) == 0:
        return None

    # 解析日期
    df['Date_parsed'] = df['Date'].apply(parse_date)
    df = df[df['Date_parsed'].notna()].copy()
    df['Date_dt'] = pd.to_datetime(df['Date_parsed'], errors='coerce')
    df = df[df['Date_dt'].notna()].sort_values('Date_dt').reset_index(drop=True)

    # 数值列
    df['FTHG'] = pd.to_numeric(df['FTHG'], errors='coerce')
    df['FTAG'] = pd.to_numeric(df['FTAG'], errors='coerce')
    df['AHh']  = pd.to_numeric(df.get('AHh', 0), errors='coerce').fillna(0)
    df['B365AHH'] = pd.to_numeric(df.get('B365AHH', 0), errors='coerce')
    df['B365AHA'] = pd.to_numeric(df.get('B365AHA', 0), errors='coerce')

    # 日期过滤：2025.1.1 - 2026.2.28
    start_dt = pd.Timestamp('2025-01-01')
    end_dt   = pd.Timestamp('2026-02-28')
    df = df[(df['Date_dt'] >= start_dt) & (df['Date_dt'] <= end_dt)].copy()

    # 必须有比分
    df = df[df['FTHG'].notna() & df['FTAG'].notna()].reset_index(drop=True)

    # 必须有有效亚盘赔率（B365AHH > 1 且不是占位符0.9）
    if 'odds_source' in df.columns and code in MERGED_LEAGUES:
        has_ah = df['B365AHH'].notna() & (df['B365AHH'] > 1.0) & (df['B365AHH'] != 0.9)
    else:
        has_ah = df['B365AHH'].notna() & (df['B365AHH'] > 1.0) & (df['B365AHH'] != 0.9)

    df_ah = df[has_ah].reset_index(drop=True)
    print(f'    过滤2025.1.1-2026.2.28: 总{df.shape[0]}场 -> 有亚盘赔率:{df_ah.shape[0]}场')

    if len(df_ah) < 50:
        return None

    return df_ah


# ============ 模型训练 ============
print("=== 亚盘让球盘模型训练 V2 (2025.1.1-2026.2.28真实赔率) ===")
print(f"目标: 75%+ 投注准确率\n")

results = []
TARGET_LEAGUES = ['d1','d2','f1','f2','cz1','sc1','e1','e2','e2n','e3','i1','sp1','tr1','p1','e0','sp2','i2','b1','g1','n1','a1','r1','br1','m1']

for code in TARGET_LEAGUES:
    print(f"\n处理 {code} ({LEAGUE_NAMES.get(code,code)})...", end=" ", flush=True)
    sys.stdout.flush()

    df = prepare_data(code)
    if df is None or len(df) < 50:
        print("数据不足")
        continue

    n = len(df)
    # 时间序列分割：前70%训练，后30%测试
    n_train = int(n * 0.70)
    train_df = df.iloc[:n_train].reset_index(drop=True)
    test_df  = df.iloc[n_train:].reset_index(drop=True)

    print(f"    train={len(train_df)} test={len(test_df)}", end=" ", flush=True)

    y_train = np.array([get_ah_outcome(train_df.iloc[i]) for i in range(len(train_df))])
    y_test  = np.array([get_ah_outcome(test_df.iloc[i])  for i in range(len(test_df))])

    # 滚动历史（用完整df构建，确保test期间历史连续）
    team_hist_full = make_hist()
    # 先用全部数据跑一遍特征构建（历史会更新），然后重新截取
    full_features = build_features(df, team_hist_full)

    # 用train长度截取
    train_hist = make_hist()
    # 重新从train数据构建（warm start）
    train_df_feat = train_df.copy()
    train_X = build_features(train_df_feat, train_hist).fillna(0).replace([np.inf,-np.inf], 0)

    # 对test：接续train的历史继续构建
    # 先用train数据初始化team_hist
    th_test = make_hist()
    # 把train数据先喂进去
    _ = build_features(train_df.copy(), th_test)
    test_df_feat = test_df.copy()
    test_X = build_features(test_df_feat, th_test).fillna(0).replace([np.inf,-np.inf], 0)

    # 对齐特征列
    common_cols = sorted(set(train_X.columns) & set(test_X.columns))
    train_X = train_X[common_cols]
    test_X  = test_X[common_cols]

    print(f"特征={len(common_cols)}", end=" ", flush=True)

    best_bet_acc = 0
    best_result = None

    # ===== 模型池 =====
    models = [
        ('LR_C0.1', LogisticRegression(max_iter=2000, C=0.1, solver='lbfgs')),
        ('LR_C0.5', LogisticRegression(max_iter=2000, C=0.5, solver='lbfgs')),
        ('LR_C1',   LogisticRegression(max_iter=2000, C=1.0, solver='lbfgs')),
        ('LR_C2',   LogisticRegression(max_iter=2000, C=2.0, solver='lbfgs')),
        ('LR_C5',   LogisticRegression(max_iter=2000, C=5.0, solver='lbfgs')),
        ('RF_d2',   RandomForestClassifier(n_estimators=300, max_depth=2,  random_state=42, min_samples_leaf=10)),
        ('RF_d3',   RandomForestClassifier(n_estimators=300, max_depth=3,  random_state=42, min_samples_leaf=8)),
        ('RF_d4',   RandomForestClassifier(n_estimators=300, max_depth=4,  random_state=42, min_samples_leaf=5)),
        ('RF_d5',   RandomForestClassifier(n_estimators=300, max_depth=5,  random_state=42)),
        ('ET_d2',   ExtraTreesClassifier(n_estimators=300, max_depth=2,  random_state=42, min_samples_leaf=10)),
        ('ET_d3',   ExtraTreesClassifier(n_estimators=300, max_depth=3,  random_state=42, min_samples_leaf=8)),
        ('ET_d4',   ExtraTreesClassifier(n_estimators=300, max_depth=4,  random_state=42, min_samples_leaf=5)),
        ('ET_d5',   ExtraTreesClassifier(n_estimators=300, max_depth=5,  random_state=42)),
        ('GB_d2',   GradientBoostingClassifier(n_estimators=150, max_depth=2, random_state=42, min_samples_leaf=10, learning_rate=0.05)),
        ('GB_d3',   GradientBoostingClassifier(n_estimators=150, max_depth=3, random_state=42, min_samples_leaf=5, learning_rate=0.05)),
        ('GB_d4',   GradientBoostingClassifier(n_estimators=150, max_depth=4, random_state=42, learning_rate=0.05)),
        ('ADA',     AdaBoostClassifier(n_estimators=100, random_state=42, learning_rate=0.5)),
        ('SVM_rbf', SVC(probability=True, C=1.0, kernel='rbf', gamma='scale')),
        ('SVM_lin', SVC(probability=True, C=1.0, kernel='linear')),
    ]

    for sname, scaler in [('std', StandardScaler()), ('rob', RobustScaler())]:
        for mname, clf in models:
            try:
                pipe = Pipeline([
                    ('scaler', scaler),
                    ('imputer', SimpleImputer(fill_value=0)),
                    ('clf', clf)
                ])
                pipe.fit(train_X.values, y_train)
                proba = pipe.predict_proba(test_X.values)[:, 1]

                # 时间序列交叉验证找最优阈值（只用训练数据）
                tscv = TimeSeriesSplit(n_splits=3)
                best_th_cv = 0.5
                best_acc_cv = 0
                for train_idx, val_idx in tscv.split(train_X):
                    X_tr, X_va = train_X.values[train_idx], train_X.values[val_idx]
                    y_tr, y_va = y_train[train_idx], y_train[val_idx]
                    pipe2 = Pipeline([
                        ('scaler', scaler),
                        ('imputer', SimpleImputer(fill_value=0)),
                        ('clf', type(clf)(**clf.get_params()))
                    ])
                    pipe2.fit(X_tr, y_tr)
                    p2 = pipe2.predict_proba(X_va)[:, 1]
                    for th in np.arange(0.20, 0.80, 0.02):
                        for side, pred_val in [('over',1), ('under',0)]:
                            mask = (p2 >= th) if side=='over' else (p2 <= th)
                            if mask.sum() >= 5:
                                acc = accuracy_score(y_va[mask], np.full(mask.sum(), pred_val))
                                if acc > best_acc_cv:
                                    best_acc_cv = best_th_cv = th

                # 在测试集上评估
                for side, pred_val in [('over', 1), ('under', 0)]:
                    for th in np.arange(max(0.15, best_th_cv - 0.10),
                                        min(0.85, best_th_cv + 0.10), 0.02):
                        mask = (proba >= th) if side == 'over' else (proba <= th)
                        if mask.sum() >= 5:
                            preds = np.full(mask.sum(), pred_val)
                            ba = accuracy_score(y_test[mask], preds)
                            if ba > best_bet_acc:
                                best_bet_acc = ba
                                best_result = {
                                    'model_name': f'{mname}_{sname}',
                                    'threshold': float(th),
                                    'side': side,
                                    'bet_count': int(mask.sum()),
                                    'bet_acc': float(ba),
                                    'cv_threshold': float(best_th_cv),
                                    'pipe': pipe
                                }
            except Exception as e:
                pass

    if best_result:
        flag = '✅ 75%+' if best_result['bet_acc'] >= 0.75 else ''
        print(f"-> {best_result['model_name']} th={best_result['threshold']:.2f} {best_result['side']} "
              f"{best_result['bet_acc']:.1%}({best_result['bet_count']}bets) {flag}")

        (OUT / code).mkdir(parents=True, exist_ok=True)
        joblib.dump(best_result['pipe'], OUT / code / 'optimized_pipeline.pkl')
        save_r = {k: v for k, v in best_result.items() if k != 'pipe'}
        save_r['league'] = code
        save_r['league_name'] = LEAGUE_NAMES.get(code, code)
        save_r['train_count'] = len(train_df)
        save_r['test_count'] = len(test_df)
        save_r['feature_count'] = len(common_cols)
        with open(OUT / code / 'optimized_config.json', 'w') as f:
            json.dump(save_r, f, ensure_ascii=False, indent=2)
        results.append(save_r)
    else:
        print("训练失败")

print("\n\n" + "="*60)
print("=== 汇总 ===")
for r in sorted(results, key=lambda x: x['bet_acc'], reverse=True):
    flag = '✅' if r['bet_acc'] >= 0.75 else ''
    print(f"  {r['league']} ({r['league_name']}): {r['bet_acc']:.1%} {flag} | "
          f"{r['model_name']} th={r['threshold']:.2f} {r['side']} {r['bet_count']}bets "
          f"(train={r['train_count']} test={r['test_count']})")

hit75 = sum(1 for r in results if r['bet_acc'] >= 0.75)
avg = sum(r['bet_acc'] for r in results) / len(results) if results else 0
print(f"\n共{len(results)}个联赛 | 达到75%+: {hit75} | 平均: {avg:.1%}")

if results:
    with open(OUT / 'ah_summary.json', 'w') as f:
        json.dump(results, f, ensure_ascii=False, indent=2)
    print(f"结果已保存到 {OUT}")
