#!/usr/bin/env python3
"""
重新训练表现不好的5个联赛 - 目标75%+
========================================
策略:
1. 只用最近2赛季数据
2. 赔率作为核心特征(有赔率的联赛)
3. 反过拟合: 限制模型复杂度, walk-forward验证
4. 高阈值投注: 只下高置信度注
"""
import json, os, sys, warnings
from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score

warnings.filterwarnings('ignore')

REPO_ROOT = Path(__file__).resolve().parents[1]
HIST_DIR  = REPO_ROOT / 'historical_data'
OUTPUT_DIR = REPO_ROOT / 'experiments' / 'goal_number_retrained'
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

RECENT_SEASONS = 2   # 只用最近2赛季
WF_TRAIN = 250
WF_TEST  = 50
WF_STEP  = 20

# 高阈值 - 只下高置信度注
TH_OVER = 0.68   # >70% 才下注大球 (a1需要)
TH_UNDER = 0.32  # <30% 才下注小球

# 也测试较低阈值
TH_ALT_OVER = 0.65
TH_ALT_UNDER = 0.35

LEAGUES = {
    'a1':  {'name': '澳超',  'has_odds': False},
    'br1': {'name': '巴甲',  'has_odds': False},
    'cz1': {'name': '捷克甲', 'has_odds': True},
    'f1':  {'name': '法甲',  'has_odds': True},
    'sc1': {'name': '苏超',  'has_odds': True},
}


def load_league(code):
    csv_path = HIST_DIR / f'{code}_history.csv'
    if not csv_path.exists():
        return None
    df = pd.read_csv(csv_path, low_memory=False)
    df['Date'] = pd.to_datetime(df['Date'], dayfirst=True, errors='coerce')
    df = df.sort_values('Date').reset_index(drop=True)
    df['FTHG'] = pd.to_numeric(df['FTHG'], errors='coerce')
    df['FTAG'] = pd.to_numeric(df['FTAG'], errors='coerce')
    df['tot'] = df['FTHG'] + df['FTAG']
    df['is_over'] = (df['tot'] > 2.5).astype(int)
    df['season_year'] = df['Date'].dt.year

    # 只留最近2个赛季
    max_year = df['Date'].max().year
    df = df[df['season_year'] >= max_year - RECENT_SEASONS].copy()

    if LEAGUES[code]['has_odds']:
        for oo, uu in [('B365>2.5','B365<2.5'), ('Avg>2.5','Avg<2.5')]:
            if oo in df.columns:
                df['over_odds'] = pd.to_numeric(df[oo], errors='coerce')
                df['under_odds'] = pd.to_numeric(df[uu], errors='coerce')
                df['mkt_over'] = df['over_odds'].apply(lambda x: 1/x if x > 1 else np.nan)
                df['mkt_und']  = df['under_odds'].apply(lambda x: 1/x if x > 1 else np.nan)
                break
    return df


def build_features(df, has_odds):
    """expanding window 特征, 无泄漏"""
    th = defaultdict(lambda: {
        'gf3':[],'ga3':[],'over3':[],
        'gf5':[],'ga5':[],'over5':[],
        'gf10':[],'ga10':[],'over10':[],
        'hgf':[],'hga':[],'hovr':[],
        'agf':[],'aga':[],'aovr':[],
    })

    rows = []
    for _, row in df.iterrows():
        home = str(row.get('HomeTeam',''))
        away = str(row.get('AwayTeam',''))
        h = th[home]; a = th[away]

        def w3(v):  return float(np.mean(v[-3:]))  if len(v)>=3 else (float(np.mean(v)) if v else np.nan)
        def w5(v):  return float(np.mean(v[-5:]))  if len(v)>=5 else (float(np.mean(v)) if v else np.nan)
        def w10(v): return float(np.mean(v[-10:])) if len(v)>=10 else (float(np.mean(v)) if v else np.nan)
        def strk(v,val=1):
            s=0
            for x in reversed(v):
                if x==val: s+=1
                else: break
            return s

        r = {
            # 近因特征
            'home_gf3': w3(h['gf3']), 'home_ga3': w3(h['ga3']),
            'away_gf3': w3(a['gf3']), 'away_ga3': w3(a['ga3']),
            'home_gf5': w5(h['gf5']), 'home_ga5': w5(h['ga5']),
            'away_gf5': w5(a['gf5']), 'away_ga5': w5(a['ga5']),
            'home_gf10': w10(h['gf10']), 'home_ga10': w10(h['ga10']),
            'away_gf10': w10(a['gf10']), 'away_ga10': w10(a['ga10']),
            # Over率
            'home_ovr3':  w3(h['over3']),  'away_ovr3':  w3(a['over3']),
            'home_ovr5':  w5(h['over5']),  'away_ovr5':  w5(a['over5']),
            'home_ovr10': w10(h['over10']),'away_ovr10': w10(a['over10']),
            # 连庄
            'home_ovr_strk': strk(h['over3'],1), 'home_und_strk': strk(h['over3'],0),
            'away_ovr_strk': strk(a['over3'],1), 'away_und_strk': strk(a['over3'],0),
            # 主场/客场
            'home_hgf': float(np.mean(h['hgf'])) if h['hgf'] else np.nan,
            'home_hga': float(np.mean(h['hga'])) if h['hga'] else np.nan,
            'away_agf': float(np.mean(a['agf'])) if a['agf'] else np.nan,
            'away_aga': float(np.mean(a['aga'])) if a['aga'] else np.nan,
            # 期望进球
            'exp_g_home': w5(h['gf5']) + w5(a['aga']),
            'exp_g_away': w5(a['gf5']) + w5(h['hga']),
            'exp_tot':    (w5(h['gf5'])+w5(h['ga5'])+w5(a['gf5'])+w5(a['ga5']))/2,
            # 差值
            'recent_ovr_diff': w5(h['over5']) - w5(a['over5']),
            'tot_diff': (w5(h['gf5'])+w5(a['aga'])) - (w5(a['gf5'])+w5(h['hga'])),
            'home_gd3':  w3(h['gf3'])-w3(h['ga3']),
            'away_gd3':  w3(a['gf3'])-w3(a['ga3']),
        }

        # 赔率特征(核心!)
        if has_odds and pd.notna(row.get('mkt_over')) and pd.notna(row.get('mkt_und')):
            mko = float(row['mkt_over'])
            mku = float(row['mkt_und'])
            fair = mko / (mko + mku)
            r['mkt_over'] = mko
            r['mkt_fair'] = fair
            r['mkt_margin'] = mko + mku - 1
            r['mkt_und'] = mku
        else:
            r['mkt_over'] = r['mkt_fair'] = 0.5
            r['mkt_margin'] = 0; r['mkt_und'] = 0.5

        r['is_over'] = int(row['is_over'])
        rows.append(r)

        # 更新
        tot = float(row['tot']); io = int(row['is_over'])
        hg  = float(row['FTHG']) if pd.notna(row.get('FTHG')) else tot
        ag  = float(row['FTAG']) if pd.notna(row.get('FTAG')) else tot

        for s in [h, a]:
            s['gf3'].append(tot); s['ga3'].append(tot); s['over3'].append(io)
            s['gf5'].append(tot); s['ga5'].append(tot); s['over5'].append(io)
            s['gf10'].append(tot);s['ga10'].append(tot);s['over10'].append(io)

        if pd.notna(row.get('FTHG')):
            h['hgf'].append(float(row['FTHG']))
            h['hga'].append(float(row['FTAG']))
            h['hovr'].append(io)
            a['agf'].append(float(row['FTAG']))
            a['aga'].append(float(row['FTHG']))
            a['aovr'].append(io)

    return pd.DataFrame(rows)


def walk_forward(X, y, model_fn, th_over, th_under):
    n = len(X)
    if n < WF_TRAIN + WF_TEST:
        return None

    all_true=[]; all_prob=[]; all_bet=[]; all_pred=[]

    for start in range(0, n-WF_TRAIN-WF_TEST+1, WF_STEP):
        tr_end=start+WF_TRAIN; te_end=tr_end+WF_TEST
        X_tr,X_te = X.iloc[start:tr_end], X.iloc[tr_end:te_end]
        y_tr,y_te = y.iloc[start:tr_end], y.iloc[tr_end:te_end]
        if len(X_tr)<100 or len(X_te)<10: continue

        scaler = StandardScaler()
        X_tr_s = scaler.fit_transform(X_tr.fillna(X_tr.median()))
        X_te_s = scaler.transform(X_te.fillna(X_tr.median()))

        model = model_fn()
        model.fit(X_tr_s, y_tr)
        probs = model.predict_proba(X_te_s)[:,1]

        # 投注逻辑
        preds = np.full(len(probs), -1, dtype=int)
        preds[probs >= th_over] = 1
        preds[probs <= th_under] = 0
        bet_mask = preds != -1

        all_true.extend(y_te.values)
        all_prob.extend(probs)
        all_bet.extend(bet_mask)
        all_pred.extend(preds)

    if not all_true:
        return None

    all_true=np.array(all_true); all_prob=np.array(all_prob)
    all_bet=np.array(all_bet); all_pred=np.array(all_pred)

    # 整体准确率
    valid_mask = all_pred != -1
    overall = accuracy_score(all_true[valid_mask], all_pred[valid_mask]) if valid_mask.sum() > 0 else 0.0

    # 投注准确率
    bet_t = all_true[all_bet]; bet_p = all_pred[all_bet]
    bet_acc = accuracy_score(bet_t, bet_p) if all_bet.sum() >= 10 else 0.0

    return {
        'overall': float(overall),
        'bet_acc': float(bet_acc),
        'bet_count': int(all_bet.sum()),
        'th_over': th_over,
        'th_under': th_under,
    }


def train_league(code):
    print(f"\n{'='*60}")
    print(f"  {LEAGUES[code]['name']}({code}) - 反过拟合训练")
    print(f"{'='*60}")

    df = load_league(code)
    if df is None: print("  [SKIP] 无数据"); return None

    has_odds = LEAGUES[code]['has_odds']
    print(f"  数据:{len(df)}场, 赛季:{df['Date'].min().date()}~{df['Date'].max().date()}")
    print(f"  大球率:{df['is_over'].mean():.1%}, 赔率:{'有' if has_odds else '无'}")

    df = df.reset_index(drop=True)

    # 赔率过滤
    if has_odds:
        df_odds = df[df['mkt_over'].notna() & (df['mkt_over']>0)].copy()
        print(f"  有赔率:{len(df_odds)}场")
    else:
        df_odds = df.copy()

    feat_df = build_features(df_odds, has_odds)
    cols = [c for c in feat_df.columns if c!='is_over']
    nan = feat_df[cols].isna().mean()
    valid = [c for c in cols if nan[c]<0.5]
    X = feat_df[valid].fillna(0)
    y = feat_df['is_over'].astype(int)

    print(f"  特征:{len(valid)}个")

    # Mar-Apr测试集
    mar_mask = (df_odds['Date'].dt.month.isin([3,4])) & (df_odds['Date'].dt.year==2026)
    mar_idx = df_odds[mar_mask].index.tolist()
    train_idx = [i for i in df_odds.index if i not in mar_idx]

    X_train = X.iloc[train_idx].reset_index(drop=True)
    y_train = y.iloc[train_idx].reset_index(drop=True)
    X_mar = X.iloc[mar_idx].reset_index(drop=True) if mar_idx else None
    y_mar = y.iloc[mar_idx].values if mar_idx else None

    print(f"  训练:{len(X_train)} | 测试Mar-Apr:{len(X_mar) if X_mar is not None else 0}场")

    # 模型
    models = [
        ('RF_d4', lambda: RandomForestClassifier(
            n_estimators=150, max_depth=4, min_samples_leaf=25,
            max_features=0.4, random_state=42, n_jobs=-1)),
        ('RF_d5', lambda: RandomForestClassifier(
            n_estimators=150, max_depth=5, min_samples_leaf=20,
            max_features=0.5, random_state=42, n_jobs=-1)),
        ('GB', lambda: GradientBoostingClassifier(
            n_estimators=100, max_depth=3, learning_rate=0.05,
            min_samples_leaf=30, subsample=0.8, random_state=42)),
        ('LR', lambda: LogisticRegression(C=0.05, max_iter=1000, random_state=42)),
    ]

    # 测试不同阈值
    thresholds = [
        (TH_OVER, TH_UNDER, '高阈值'),
        (TH_ALT_OVER, TH_ALT_UNDER, '中阈值'),
        (0.65, 0.35, '标准阈值'),
    ]

    best=None

    for th_over, th_under, th_name in thresholds:
        print(f"\n  阈值: {th_name}(>{th_over}/<{th_under})")
        for name, fn in models:
            r = walk_forward(X_train, y_train, fn, th_over, th_under)
            if r and r['bet_count'] >= 10:
                print(f"    {name}: 投注={r['bet_acc']:.1%}(n={r['bet_count']})")
                if best is None or r['bet_acc'] > best['bet_acc']:
                    best = {**r, 'model_name': name, 'model_fn': fn, 'th_name': th_name}

    if best is None:
        print("  [FAIL]"); return None

    print(f"\n  最佳: {best['model_name']} {best['th_name']} 投注={best['bet_acc']:.1%}(n={best['bet_count']})")

    # 训练最终模型
    scaler = StandardScaler()
    X_full_s = scaler.fit_transform(X.fillna(X.median()))
    final = best['model_fn']()
    final.fit(X_full_s, y)

    # Mar-Apr评估
    mar_results = None
    if X_mar is not None and len(X_mar) > 0:
        Xm_s = scaler.transform(X_mar.fillna(X.median()))
        probs = final.predict_proba(Xm_s)[:,1]
        preds = np.full(len(probs), -1, dtype=int)
        preds[probs >= best['th_over']] = 1
        preds[probs <= best['th_under']] = 0
        bet_mask = preds != -1

        # 整体:只看有投注的那些
        valid_mask = preds != -1
        if valid_mask.sum() > 0:
            mar_valid = y_mar[valid_mask]
            mar_pred_valid = preds[valid_mask]
            mar_overall = accuracy_score(mar_valid, mar_pred_valid)
        else:
            mar_overall = 0.0

        if bet_mask.sum() >= 3:
            mar_bet_acc = accuracy_score(y_mar[bet_mask], preds[bet_mask])
        else:
            mar_bet_acc = 0.0

        print(f"\n  Mar-Apr评估({len(X_mar)}场):")
        print(f"    投注准确率: {mar_bet_acc:.1%}(n={int(bet_mask.sum())})")
        print(f"    整体准确率: {mar_overall:.1%}(n={int(valid_mask.sum())})")

        mar_results = {'bet_acc': mar_bet_acc, 'bet_count': int(bet_mask.sum()), 'overall': mar_overall}

        # 详细
        mar_df = df_odds.iloc[mar_idx].reset_index(drop=True)
        mar_df['prob'] = probs
        mar_df['pred'] = preds
        mar_df['bet'] = bet_mask
        mar_df['correct'] = (mar_df['pred']==mar_df['is_over']).astype(int)

        print(f"\n  详细预测:")
        for _, r2 in mar_df.iterrows():
            m = '✓' if (r2['pred']!=-1 and r2['correct']) else ('⚠' if r2['pred']==-1 else '✗')
            b = '🔴' if r2['bet'] else '  '
            tot = int(r2['FTHG'])+int(r2['FTAG']) if pd.notna(r2['FTHG']) else '?'
            pl = '大' if r2['pred']==1 else ('小' if r2['pred']==0 else '-')
            print(f"    {b}{m} {str(r2['Date'])[:10]} {str(r2['HomeTeam'])[:12]:12s} vs {str(r2['AwayTeam'])[:12]:12s} "
                  f"| prob={r2['prob']:.2f} {pl} | {tot}g {'大' if r2['is_over'] else '小'}")

    # 保存
    out = OUTPUT_DIR/code; out.mkdir(parents=True, exist_ok=True)
    joblib.dump({
        'model': final, 'scaler': scaler, 'features': valid,
        'th_over': best['th_over'], 'th_under': best['th_under'],
        'league': code, 'league_name': LEAGUES[code]['name'],
    }, out/'model.pkl')

    config = {
        'league': code, 'league_name': LEAGUES[code]['name'],
        'th_over': best['th_over'], 'th_under': best['th_under'],
        'th_name': best['th_name'],
        'features': valid, 'feature_count': len(valid),
        'model_name': best['model_name'],
        'wf_bet_acc': float(best['bet_acc']),
        'wf_bet_count': int(best['bet_count']),
        'wf_overall': float(best['overall']),
        'mar_apr_bet_acc': float(mar_results['bet_acc']) if mar_results else None,
        'mar_apr_bet_count': int(mar_results['bet_count']) if mar_results else 0,
    }
    with open(out/'config.json','w',encoding='utf-8') as f:
        json.dump(config, f, ensure_ascii=False, indent=2)

    return config


def main():
    codes = sys.argv[1:] if len(sys.argv)>1 else list(LEAGUES.keys())
    results=[]
    for code in codes:
        if code in LEAGUES:
            r = train_league(code)
            results.append(r)

    print(f"\n\n{'='*60}")
    for r in results:
        if r:
            acc = r.get('mar_apr_bet_acc') or r.get('wf_bet_acc', 0)
            s = '✓达标' if acc >= 0.75 else '✗未达标'
            src = 'Mar-Apr' if r.get('mar_apr_bet_acc') is not None else 'WF'
            print(f"  {r['league_name']:6s}: {src}={acc:.1%} {s} (th={r['th_name']}, model={r['model_name']})")


if __name__ == '__main__':
    main()
