#!/usr/bin/env python3
"""
pro_v8: 每个联赛独立最优模型（per-league AutoML）
- 每个联赛独立搜索：最优特征子集 × 最优模型超参
- 不会因某个特征对A联赛好而对B联赛差
- 策略：先跑基础特征+基础模型建立baseline，再尝试加特征/调参
输出: artifacts/v4_models_serving_pro_v8/
"""
import os, sys, json, warnings, joblib
from pathlib import Path
from collections import defaultdict
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.calibration import CalibratedClassifierCV
warnings.filterwarnings('ignore')
np.random.seed(42)

try:
    from xgboost import XGBClassifier
    USE_XGB = True
except ImportError:
    USE_XGB = False

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
ODDS_DIR = REPO / 'artifacts' / 'odds_api_history'
OUT_DIR = REPO / 'artifacts' / 'v4_models_serving_pro_v8'
OUT_DIR.mkdir(parents=True, exist_ok=True)

LEAGUE_NAMES = {
    'e0':'英超','e1':'英甲','e2':'英冠','e2n':'英乙',
    'd1':'德甲','d2':'德乙','f1':'法甲','f2':'法乙',
    'i1':'意甲','i2':'意乙','sp1':'西甲','sp2':'西乙',
    'tr1':'土超','p1':'葡超','b1':'比甲','g1':'希腊超',
    'n1':'荷兰甲','n2':'荷兰乙','sc1':'苏超','cz1':'捷甲',
    'a1':'澳超','r1':'俄超','br1':'巴甲','m1':'美职联','j1':'日职联',
}

# ═══════════════════════════════════════════════════════════════════
# 特征构建（分模块，可按需启用）
# ═══════════════════════════════════════════════════════════════════

def build_team_form(hist_df):
    cache = defaultdict(lambda: {
        'gf':[],'ga':[],'win':[],'und':[],'draw':[],'mom':[],'rf':[],
        'home_gf':[],'home_ga':[],'home_win':[],
        'away_gf':[],'away_ga':[],'away_win':[],
    })
    for _, r in hist_df.iterrows():
        ht, at = str(r.get('HomeTeam','')).strip(), str(r.get('AwayTeam','')).strip()
        fthg, ftag = r.get('FTHG'), r.get('FTAG')
        if fthg is None or ftag is None: continue
        try:
            fthg, ftag = float(fthg), float(ftag)
        except: continue
        win_h = 1 if fthg > ftag else 0
        drw = 1 if fthg == ftag else 0
        c = cache[ht]
        c['gf'].append(float(fthg)); c['ga'].append(float(ftag))
        c['win'].append(float(win_h)); c['und'].append(1.0-float(win_h))
        c['draw'].append(float(drw)); c['mom'].append(float(win_h)); c['rf'].append(float(fthg)-float(ftag))
        c['home_gf'].append(float(fthg)); c['home_ga'].append(float(ftag)); c['home_win'].append(float(win_h))
        ca = cache[at]
        ca['gf'].append(float(ftag)); ca['ga'].append(float(fthg))
        ca['win'].append(1.0-float(win_h)); ca['und'].append(float(win_h))
        ca['draw'].append(float(drw)); ca['mom'].append(1.0-float(win_h)); ca['rf'].append(float(ftag)-float(fthg))
        ca['away_gf'].append(float(ftag)); ca['away_ga'].append(float(fthg)); ca['away_win'].append(1.0-float(win_h))
    return cache

def build_h2h_cache(hist_df):
    h2h = defaultdict(list)
    for _, r in hist_df.iterrows():
        ht = str(r.get('HomeTeam','')).strip()
        at = str(r.get('AwayTeam','')).strip()
        fthg, ftag = r.get('FTHG'), r.get('FTAG')
        if fthg is None or ftag is None: continue
        try:
            fthg, ftag = float(fthg), float(ftag)
        except: continue
        h2h[(ht,at)].append({'win_h': 1 if fthg > ftag else 0, 'gf': fthg, 'ga': ftag})
    return h2h

def _mn(v, n=10):
    vals = v[-n:]; return sum(vals)/len(vals) if vals else 0.5

def _arr(v, n=10):
    return ([0.5]*n + list(v))[-n:]

def _std(v, n=10):
    vals = v[-n:]
    if len(vals) < 2: return 0.0
    m = sum(vals)/len(vals)
    return (sum((x-m)**2 for x in vals)/len(vals))**0.5

def _recent_form(v, n=10):
    vals = v[-n:]
    if len(vals) < 3: return 0.0
    return sum(vals[-3:])/3 - sum(vals)/len(vals)

def _streak(v, n=3):
    vals = v[-n:] if len(v) >= n else v
    if len(vals) < 2: return 0.0
    if all(x >= 0.5 for x in vals): return 0.8
    if all(x < 0.5 for x in vals): return -0.8
    return 0.0

def _h2h_wr(h2h, home, away, n=5):
    matches = h2h.get((home, away), [])
    if not matches: return 0.5
    recent = matches[-n:]
    return sum(m['win_h'] for m in recent) / len(recent)

def build_full_features(ahh, b365hh, b365ha, home, away, tc, h2h):
    """
    构建全量特征（54维），后续按特征组选择
    """
    c = tc.get(home, {})
    ca = tc.get(away, {})

    try:
        p_home = 1.0 / float(b365hh) if float(b365hh) > 1.0 else 0.5
        p_away = 1.0 / float(b365ha) if float(b365ha) > 1.0 else 0.5
    except:
        p_home = p_away = 0.5

    prob_sum = p_home + p_away
    prob_diff = p_home - p_away
    water_home = float(b365hh) - (prob_sum / p_home) if p_home > 0 else 0
    water_away = float(b365ha) - (prob_sum / p_away) if p_away > 0 else 0

    # ── 基础特征组 ──
    h_wr10 = _mn(c.get('win',[]), 10); a_wr10 = _mn(ca.get('win',[]), 10)
    h_wr3 = _mn(c.get('win',[]), 3); a_wr3 = _mn(ca.get('win',[]), 3)
    h_mom = _mn(c.get('mom',[]), 5); a_mom = _mn(ca.get('mom',[]), 5)
    h_recent = _recent_form(c.get('win',[]), 10); a_recent = _recent_form(ca.get('win',[]), 10)
    h_var = _std(c.get('win',[]), 10); a_var = _std(ca.get('win',[]), 10)
    h_gd10 = _mn(c.get('rf',[]), 10); a_gd10 = _mn(ca.get('rf',[]), 10)
    h_gf10 = _mn(c.get('gf',[]), 10); a_gf10 = _mn(ca.get('gf',[]), 10)
    h_ga10 = _mn(c.get('ga',[]), 10); a_ga10 = _mn(ca.get('ga',[]), 10)
    rec = _arr(c.get('win',[]), 10) + _arr(ca.get('win',[]), 10)

    base_feat = [
        float(ahh), p_home, p_away, prob_diff,
        water_home, water_away, float(ahh) * prob_diff,
        h_wr10 - a_wr10, h_wr3 - a_wr3, h_mom - a_mom,
        h_recent - a_recent, h_var, a_var,
        h_gd10, a_gd10, h_gf10 - a_gf10, h_ga10 - a_ga10,
    ] + rec

    # ── 高级特征组1：H2H ──
    h2h_feat = [
        _h2h_wr(h2h, home, away, 5),
        _h2h_wr(h2h, home, away, 10),
    ]

    # ── 高级特征组2：主客场特定 ──
    h_home_wr5 = _mn(c.get('home_win',[]), 5)
    a_away_wr5 = _mn(ca.get('away_win',[]), 5)
    h_home_gf5 = _mn(c.get('home_gf',[]), 5)
    a_away_ga5 = _mn(ca.get('away_ga',[]), 5)
    home_adv_feat = [
        h_home_wr5 - a_away_wr5,
        h_home_gf5 - a_away_ga5,
        _mn(c.get('home_gf',[]), 5),
        _mn(ca.get('away_ga',[]), 5),
    ]

    # ── 高级特征组3：连红/连黑/趋势 ──
    streak_feat = [
        _streak(c.get('win',[]), 3),
        _streak(ca.get('win',[]), 3),
        _streak(c.get('home_win',[]), 3),
        _streak(ca.get('away_win',[]), 3),
    ]

    # ── 高级特征组4：赔率×实力交互 ──
    interact_feat = [
        p_home * (h_wr10 - 0.5),
        float(ahh) * (h_wr10 - a_wr10),
        prob_diff * (h_wr10 - a_wr10),
    ]

    # ── 高级特征组5（P0改进）：Pinnacle市场特征 ──
    # 这些特征从预测时的 _extract_pinnacle_ah() 获取
    # 训练时从历史CSV的 PinnAHh_closing/PinnSteam 列读取（如果有）
    market_feat = []

    return {
        'base': base_feat,
        'h2h': h2h_feat,
        'home_adv': home_adv_feat,
        'streak': streak_feat,
        'interact': interact_feat,
        'market': market_feat,
    }

def concat_features(feature_dict, groups):
    """按组拼接特征"""
    result = []
    for g in groups:
        result.extend(feature_dict.get(g, []))
    return result

# ═══════════════════════════════════════════════════════════════════
# 特征组配置（每个联赛的候选特征集）
# ═══════════════════════════════════════════════════════════════════

FEATURE_CONFIGS = [
    # config 0: 仅基础特征（最保守，防止过拟合）
    ['base'],
    # config 1: 基础 + H2H
    ['base', 'h2h'],
    # config 2: 基础 + 主客场
    ['base', 'home_adv'],
    # config 3: 基础 + H2H + 主客场
    ['base', 'h2h', 'home_adv'],
    # config 4: 全量（基础 + H2H + 主客场 + 连红 + 交互）
    ['base', 'h2h', 'home_adv', 'streak', 'interact'],
    # config 5: 基础 + 连红
    ['base', 'streak'],
    # config 6: 基础 + 交互
    ['base', 'interact'],
]

# 模型超参配置（每个联赛的候选模型）
MODEL_CONFIGS = [
    # (name, model_fn)
    ('RF_d4', lambda: RandomForestClassifier(n_estimators=100, max_depth=4, min_samples_leaf=10, random_state=42, n_jobs=-1)),
    ('RF_d6', lambda: RandomForestClassifier(n_estimators=150, max_depth=6, min_samples_leaf=8, random_state=42, n_jobs=-1)),
    ('RF_d8', lambda: RandomForestClassifier(n_estimators=200, max_depth=8, min_samples_leaf=5, random_state=42, n_jobs=-1)),
    ('GBT_d3', lambda: GradientBoostingClassifier(n_estimators=100, max_depth=3, learning_rate=0.05, random_state=42)),
    ('GBT_d5', lambda: GradientBoostingClassifier(n_estimators=150, max_depth=5, learning_rate=0.05, random_state=42)),
]

if USE_XGB:
    MODEL_CONFIGS += [
        ('XGB_d4', lambda: XGBClassifier(n_estimators=150, max_depth=4, learning_rate=0.05, subsample=0.8, colsample_bytree=0.8, random_state=42, n_jobs=-1, eval_metric='logloss', use_label_encoder=False)),
        ('XGB_d6', lambda: XGBClassifier(n_estimators=200, max_depth=6, learning_rate=0.05, subsample=0.8, colsample_bytree=0.8, random_state=42, n_jobs=-1, eval_metric='logloss', use_label_encoder=False)),
        ('XGB_d5_cons', lambda: XGBClassifier(n_estimators=150, max_depth=5, learning_rate=0.03, subsample=0.7, colsample_bytree=0.7, reg_alpha=0.1, reg_lambda=1.0, random_state=42, n_jobs=-1, eval_metric='logloss', use_label_encoder=False)),
    ]

# ═══════════════════════════════════════════════════════════════════
# 训练单个联赛：搜索最佳特征集+模型组合
# ═══════════════════════════════════════════════════════════════════

def train_league(code, all_features_list, labels):
    """
    对单个联赛搜索最佳 (特征配置, 模型配置)
    all_features_dict: {idx: feature_vector}  对应每条数据的特征
    labels: array of 0/1
    """
    n = len(labels)
    if n < 80:
        return None, None, None, f'数据太少({n})'

    split = max(50, int(n * 0.8))
    indices = np.random.permutation(n)
    train_idx, test_idx = indices[:split], indices[split:]
    
    y = np.array(labels)
    y_tr, y_te = y[train_idx], y[test_idx]

    best_acc = 0
    best_config = None
    best_model = None
    best_feat_dim = 0
    results_log = []

    for fidx, feat_groups in enumerate(FEATURE_CONFIGS):
        feat_dim = sum(len(all_features_list[0].get(g, [])) for g in feat_groups)
        # 构建该特征集
        X_tr_f = np.array([concat_features(all_features_list[i], feat_groups) for i in train_idx], dtype=float)
        X_te_f = np.array([concat_features(all_features_list[i], feat_groups) for i in test_idx], dtype=float)
        
        if X_tr_f.shape[1] == 0:
            continue

        for mname, model_fn in MODEL_CONFIGS:
            try:
                model = model_fn()
                model.fit(X_tr_f, y_tr)
                acc = accuracy_score(y_te, model.predict(X_te_f))
                
                # 校准验证
                try:
                    cal = CalibratedClassifierCV(model, cv=3, method='isotonic')
                    cal.fit(X_tr_f, y_tr)
                    cal_acc = accuracy_score(y_te, cal.predict(X_te_f))
                except:
                    cal_acc = acc
                    cal = model

                results_log.append({
                    'feat_idx': fidx, 'model': mname,
                    'acc': acc, 'cal_acc': cal_acc,
                    'feat_dim': X_tr_f.shape[1], 'feat_groups': '+'.join(feat_groups)
                })

                if cal_acc > best_acc:
                    best_acc = cal_acc
                    best_config = (fidx, feat_groups, mname)
                    best_model = cal
                    best_feat_dim = X_tr_f.shape[1]
            except Exception as e:
                continue

    if best_model is None:
        return None, None, None, '全部配置失败'

    # 用全部训练数据重新拟合（不用hold-out）
    X_full_f = np.array([concat_features(all_features_list[i], best_config[1]) for i in range(n)], dtype=float)
    scaler = StandardScaler()
    scaler.fit(X_full_f)
    X_tr_s = scaler.transform(X_full_f)
    best_model.fit(X_tr_s, y)

    return best_model, scaler, {
        'acc': best_acc,
        'feat_idx': best_config[0],
        'feat_groups': '+'.join(best_config[1]),
        'model_name': best_config[2],
        'n_features': best_feat_dim,
        'n_samples': n,
        'league': code,
        'search_log': results_log,
    }, None


def prepare_league_data(code):
    """加载并构建单个联赛的全量特征（分组的）"""
    full_file = ODDS_DIR / f'{code}_full_odds.csv'
    if not full_file.exists():
        return None, None, None, f'无文件'

    df = pd.read_csv(full_file, low_memory=False)
    
    ahh_col = 'PinnAHh' if 'PinnAHh' in df.columns else 'AHh'
    bhh_col = 'PinnAHH' if 'PinnAHH' in df.columns else 'AHH'
    bha_col = 'PinnAHA' if 'PinnAHA' in df.columns else 'AHA'

    df = df[df[ahh_col].notna() & df[bhh_col].notna() & (df[bhh_col] > 1.0) & df[bha_col].notna() & (df[bha_col] > 1.0)]
    df = df[df['FTHG'].notna() & df['FTAG'].notna()]
    if len(df) < 80:
        return None, None, None, f'数据太少({len(df)})'

    df['label'] = (df['FTHG'] + df[ahh_col] > df['FTAG']).astype(int)

    hist_files = list(ODDS_DIR.glob(f'{code}_S*_fixtures.csv'))
    if not hist_files:
        return None, None, None, f'无历史数据'
    all_hist = pd.concat([pd.read_csv(f, usecols=['HomeTeam','AwayTeam','FTHG','FTAG'], low_memory=False) for f in hist_files], ignore_index=True)
    tc = build_team_form(all_hist)
    h2h = build_h2h_cache(all_hist)

    all_features = []
    y_rows = []
    for _, r in df.iterrows():
        try:
            feat = build_full_features(float(r[ahh_col]), float(r[bhh_col]), float(r[bha_col]),
                                      str(r['HomeTeam']), str(r['AwayTeam']), tc, h2h)
            all_features.append(feat)
            y_rows.append(int(r['label']))
        except:
            continue

    if len(all_features) < 80:
        return None, None, None, f'特征构建后不足({len(all_features)})'

    return all_features, np.array(y_rows), tc, None


def train_and_select_best(code):
    """训练单个联赛：准备数据 → 搜索最优 → 保存"""
    print(f"\n{'='*50}")
    print(f"训练 {code} ({LEAGUE_NAMES.get(code, code)})")
    print(f"{'='*50}")
    
    all_features, y, tc, err = prepare_league_data(code)
    if err:
        print(f"  → {err}")
        return None, err

    print(f"  样本数: {len(y)}, 特征组: base(26) + h2h(2) + home_adv(4) + streak(4) + interact(3)")
    print(f"  模型候选: {len(MODEL_CONFIGS)}种 × {len(FEATURE_CONFIGS)}种特征 = {len(MODEL_CONFIGS)*len(FEATURE_CONFIGS)}种组合")
    
    # 把数据组织成 (n_samples, n_feature_groups) 格式用于搜索
    # all_features 已经是 {group: [...] } 格式
    model, scaler, info, _ = train_league(code, all_features, y)
    
    if model is None:
        print(f"  → {info}")
        return None, info

    print(f"  最优: feat={info['feat_groups']}, model={info['model_name']}, acc={info['acc']:.1%}, n_feat={info['n_features']}")
    
    # 保存
    out_file = OUT_DIR / f'{code}_pro_v8_model.pkl'
    joblib.dump({
        'model': model, 'scaler': scaler,
        'acc': info['acc'],
        'feat_groups': info['feat_groups'],
        'model_name': info['model_name'],
        'n_features': info['n_features'],
        'n_samples': info['n_samples'],
        'league': code,
    }, out_file)

    # 保存该联赛的搜索日志
    log_file = OUT_DIR / f'{code}_search_log.json'
    with open(log_file, 'w') as f:
        json.dump(info['search_log'], f, indent=2)

    return info['acc'], info['n_samples']


# ═══════════════════════════════════════════════════════════════════
# 主流程
# ═══════════════════════════════════════════════════════════════════

if __name__ == '__main__':
    print("=" * 60)
    print("pro_v8 训练开始（per-league AutoML 独立最优）")
    print("=" * 60)
    print(f"XGBoost可用: {USE_XGB}")
    print(f"特征组: base, h2h, home_adv, streak, interact")
    print(f"模型: {', '.join(m for m,_ in MODEL_CONFIGS)}")

    results = {}
    for code in sorted(LEAGUE_NAMES.keys()):
        try:
            acc, n = train_and_select_best(code)
            if acc is not None:
                results[code] = {'acc': acc, 'n': n, 'status': '✅成功'}
            else:
                results[code] = {'status': f'⚠️{n}'}
        except Exception as e:
            import traceback; traceback.print_exc()
            results[code] = {'status': f'❌{e}'}

    # 汇总
    print("\n" + "=" * 70)
    print("pro_v8 训练结果汇总")
    print("=" * 70)
    print(f"{'联赛':<6} {'名称':<10} {'准确率':<10} {'样本':<6} {'特征组':<30} {'模型':<15} {'状态'}")
    print("-" * 100)
    for code, r in sorted(results.items()):
        name = LEAGUE_NAMES.get(code, code)
        if 'acc' in r:
            print(f"{code:<6} {name:<10} {r['acc']:.1%}       {r['n']:<6}")
        else:
            print(f"{code:<6} {name:<10} {'-':<10} {'-':<6}  {r.get('status','N/A')}")

    summary = {code: {'league_name': LEAGUE_NAMES.get(code, code), **r}
                for code, r in results.items()}
    with open(OUT_DIR / 'pro_v8_summary.json', 'w') as f:
        json.dump(summary, f, indent=2, ensure_ascii=False)
    print(f"\n模型保存: {OUT_DIR}")
