#!/usr/bin/env python3
"""
亚盘模型 v5 - 精调版
- 2024-2025数据（去除2023旧数据减少分布偏移）
- 移除calendar噪声（month, dow）
- Per-league阈值优化
- Isotonic校准
- 更强正则化减少过拟合
- 只在模型有优势时使用，否则用庄家方向
"""
from __future__ import annotations
import os, sys, json, warnings
from collections import defaultdict
from pathlib import Path
from datetime import datetime
import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier, HistGradientBoostingClassifier
from sklearn.calibration import CalibratedClassifierCV
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedKFold

warnings.filterwarnings('ignore')
try:
    import lightgbm as lgb_lib
    HAS_LGB = True
except:
    HAS_LGB = False
try:
    import xgboost as xgb_lib
    HAS_XGB = True
except:
    HAS_XGB = False

REPO = Path("/root/.openclaw/workspace-flj_bot1/football-predictor-repro")
HIST = REPO / "historical_data"
OUT = REPO / "models/ah_models_v5"
OUT.mkdir(parents=True, exist_ok=True)

LNAME = {
    "e2":"英冠","e3":"英乙","e1":"英甲","d1":"德甲",
    "f1":"法甲","i1":"意甲","sp1":"西甲","sc1":"苏超",
    "tr1":"土超","p1":"葡超","e2n":"荷乙","cz1":"捷克甲",
}


def pr(odds):
    try:
        v = float(odds)
        return 1.0 / v if v > 1 else np.nan
    except:
        return np.nan


def avg5(v):
    if not v:
        return 2.5
    v = list(v)
    return float(np.mean(v[-5:] if len(v) >= 5 else v)) if v else 2.5


def avg3(v):
    if not v:
        return 2.5
    v = list(v)
    return float(np.mean(v[-3:] if len(v) >= 3 else v)) if v else 2.5


def wr(v):
    if not v:
        return 0.5
    return float(np.mean(list(v)))


def strk(v, t):
    s = 0
    for x in reversed(list(v) if v else []):
        if x == t:
            s += 1
        else:
            break
    return s


def load_data(code, years=(2024, 2025)):
    """加载数据，默认只用2024-2025"""
    hp = HIST / (code + "_history.csv")
    if not hp.exists():
        return pd.DataFrame()
    df = pd.read_csv(hp, low_memory=False)
    df["Date"] = pd.to_datetime(df["Date"], errors="coerce", dayfirst=True)
    if years == (2024, 2025):
        df = df[(df["Date"] >= "2024-01-01") & (df["Date"] <= "2025-12-31")]
    else:
        df = df[(df["Date"] >= "2023-01-01") & (df["Date"] <= "2025-12-31")]
    needed = ["FTHG", "FTAG", "AHh", "B365AHH", "B365AHA"]
    df = df.dropna(subset=needed)
    df = df[df["B365AHH"].astype(float) > 1].copy().reset_index(drop=True)
    df["gd"] = df["FTHG"].astype(float) - df["FTAG"].astype(float)
    df["gd_ah"] = df["gd"] + df["AHh"].astype(float)
    df = df[df["gd_ah"].abs() >= 0.01].reset_index(drop=True)
    df["target"] = (df["gd_ah"] > 0).astype(int)
    bm_ph = df["B365AHH"].apply(pr)
    bm_pa = df["B365AHA"].apply(pr)
    ov_sum = bm_ph + bm_pa
    df["bm_ph_n"] = bm_ph / ov_sum
    df["bm_pa_n"] = bm_pa / ov_sum
    df["bm_dir"] = (df["bm_ph_n"] > 0.5).astype(int)
    df["bm_conf"] = (df["bm_ph_n"] - 0.5).abs() * 2
    df["bm_edge"] = df["bm_ph_n"] - 0.5
    df["bm_bal"] = (df["B365AHH"].astype(float) - df["B365AHA"].astype(float)).abs()
    if "B365H" in df.columns:
        bh = df["B365H"].apply(pr)
        ba = df["B365A"].apply(pr)
        df["bm_3w_dir"] = (bh > ba).astype(int)
    else:
        df["bm_3w_dir"] = 0
    return df


# V5: 精简特征集，移除calendar噪声
V5_FEATURES = [
    # 庄家核心
    "bm_prob_home", "bm_prob_away", "bm_edge", "bm_conf",
    "bm_balance", "bm_dir", "bm_3w_dir",
    # AH线特征
    "ahh", "ahh_abs",
    "ah_strong", "ah_medium", "ah_weak",
    "ah_away", "ah_home",
    # AH历史（最强信号）
    "bm_vs_ht10", "bm_vs_at10", "bm_vs_lg",
    "ht_ah10", "at_ah10", "ht_ah20", "at_ah20",
    "ht_ah_strk", "at_ah_strk",
    "ht_home_ah", "at_away_ah",
    "ht_ah_L3", "at_ah_L3", "ht_ah_L5", "at_ah_L5",
    # 进攻/防守
    "ht_gf_avg", "ht_ga_avg", "at_gf_avg", "at_ga_avg",
    "ht_tot_avg", "at_tot_avg",
    "ht_ov10", "at_ov10",
    "ht_gf5", "at_gf5", "ht_ov5", "at_ov5",
    "form_diff", "eg_diff", "att_diff", "def_diff",
    # 价值指标
    "lg_ah", "lg_tot", "lg_ov10",
    "bm_ht_val", "bm_at_val", "bm_lg_val",
    # 交互
    "conf_ht_interact", "eg_ah_interact", "ht_at_ah_diff",
    # 市场信号
    "mkt_align",
]


def build_features(df):
    ts = defaultdict(lambda: {
        "gf": [], "ga": [], "tot": [], "ov": [],
        "hm_gf": [], "hm_ga": [], "ah_h": [], "ah_a": [],
    })
    lg = {"tot": [], "ov": [], "ah_h": []}
    Xl = []
    yl = []
    info = []
    WARM = 8

    for i, row in df.iterrows():
        hm = str(row["HomeTeam"])
        aw = str(row["AwayTeam"])
        hg = float(row["FTHG"])
        ag = float(row["FTAG"])
        ahh = float(row["AHh"])
        bm_ph = float(row["bm_ph_n"])
        bm_pa = float(row["bm_pa_n"])
        bm_dir = int(row["bm_dir"])
        bm_3w = int(row.get("bm_3w_dir", 0))

        if i < WARM:
            _upd(ts, lg, hm, aw, hg, ag, ahh)
            continue

        h = ts.get(hm, {})
        a = ts.get(aw, {})
        lg_ah = wr(lg["ah_h"][-50:]) if lg["ah_h"] else 0.5
        lg_ov10 = wr(lg["ov"][-10:]) if lg["ov"] else 0.5
        lg_tot_v = float(np.mean(lg["tot"])) if lg["tot"] else 2.5
        ht_ah10 = wr(h.get("ah_h", [])[-10:] if h.get("ah_h") else [])
        at_ah10 = wr(a.get("ah_a", [])[-10:] if a.get("ah_a") else [])
        ht_ah20 = wr(h.get("ah_h", [])[-20:] if h.get("ah_h") else [])
        at_ah20 = wr(a.get("ah_a", [])[-20:] if a.get("ah_a") else [])
        ht_gf = avg5(h.get("gf", [])); at_gf = avg5(a.get("gf", []))
        ht_ga = avg5(h.get("ga", [])); at_ga = avg5(a.get("ga", []))
        eg_diff = ht_gf - at_ga

        f = {}
        f["bm_prob_home"] = bm_ph
        f["bm_prob_away"] = bm_pa
        f["bm_edge"] = bm_ph - 0.5
        f["bm_conf"] = abs(bm_ph - 0.5) * 2
        f["bm_balance"] = float(row.get("bm_bal", 0))
        f["bm_dir"] = bm_dir
        f["ahh"] = ahh
        f["ahh_abs"] = abs(ahh)
        f["ah_strong"] = 1.0 if abs(ahh) >= 1.0 else 0.0
        f["ah_medium"] = 1.0 if 0.25 <= abs(ahh) < 1.0 else 0.0
        f["ah_weak"] = 1.0 if abs(ahh) < 0.25 else 0.0
        f["ah_away"] = 1.0 if ahh > 0 else 0.0
        f["ah_home"] = 1.0 if ahh < 0 else 0.0
        f["bm_vs_ht10"] = float(bm_dir) - ht_ah10
        f["bm_vs_at10"] = float(1 - bm_dir) - at_ah10
        f["bm_vs_lg"] = bm_ph - lg_ah
        f["ht_ah10"] = ht_ah10
        f["at_ah10"] = at_ah10
        f["ht_ah20"] = ht_ah20
        f["at_ah20"] = at_ah20
        f["ht_ah_strk"] = strk(h.get("ah_h", []), 1)
        f["at_ah_strk"] = strk(a.get("ah_a", []), 1)
        f["ht_home_ah"] = wr(h.get("ah_h", [])[-10:] if h.get("ah_h") else [])
        f["at_away_ah"] = wr(a.get("ah_a", [])[-10:] if a.get("ah_a") else [])
        f["ht_gf_avg"] = ht_gf
        f["ht_ga_avg"] = ht_ga
        f["at_gf_avg"] = at_gf
        f["at_ga_avg"] = at_ga
        f["ht_tot_avg"] = avg5(h.get("tot", []))
        f["at_tot_avg"] = avg5(a.get("tot", []))
        f["ht_ov10"] = wr(h.get("ov", [])[-10:] if h.get("ov") else [])
        f["at_ov10"] = wr(a.get("ov", [])[-10:] if a.get("ov") else [])
        f["ht_gf5"] = avg3(h.get("gf", []))
        f["at_gf5"] = avg3(a.get("gf", []))
        f["ht_ov5"] = wr(h.get("ov", [])[-5:] if h.get("ov") else [])
        f["at_ov5"] = wr(a.get("ov", [])[-5:] if h.get("ov") else [])
        f["form_diff"] = wr(h.get("ov", [])[-5:]) - wr(a.get("ov", [])[-5:]) if (h.get("ov") and a.get("ov")) else 0
        f["eg_diff"] = eg_diff
        f["att_diff"] = ht_gf - at_gf
        f["def_diff"] = at_ga - ht_ga
        f["lg_ah"] = lg_ah
        f["lg_tot"] = lg_tot_v
        f["lg_ov10"] = lg_ov10
        f["bm_3w_dir"] = bm_3w
        f["mkt_align"] = 1.0 if bm_dir == bm_3w else 0.0
        f["bm_ht_val"] = bm_ph - ht_ah10
        f["bm_at_val"] = bm_pa - at_ah10
        f["bm_lg_val"] = bm_ph - lg_ah
        f["conf_ht_interact"] = abs(bm_ph - 0.5) * 2 * ht_ah10
        f["eg_ah_interact"] = eg_diff * ht_ah10
        f["ht_at_ah_diff"] = ht_ah10 - at_ah10
        f["ht_ah_L3"] = float(sum(list(h.get("ah_h", [])[-3:]) if h.get("ah_h") else []))
        f["at_ah_L3"] = float(sum(list(a.get("ah_a", [])[-3:]) if a.get("ah_a") else []))
        f["ht_ah_L5"] = float(sum(list(h.get("ah_h", [])[-5:]) if h.get("ah_h") else []))
        f["at_ah_L5"] = float(sum(list(a.get("ah_a", [])[-5:]) if a.get("ah_a") else []))

        Xl.append(f)
        yl.append(int(row["target"]))
        info.append({"bm_prob": bm_ph, "bm_dir": bm_dir, "ahh": ahh})
        _upd(ts, lg, hm, aw, hg, ag, ahh)

    return pd.DataFrame(Xl).fillna(0), np.array(yl), info


def _upd(ts, lg, hm, aw, hg, ag, ahh):
    tot = hg + ag
    ov = int(tot > 2.5)
    ahw = int(hg - ag + ahh > 0)
    h = ts[hm]; a = ts[aw]
    h["gf"].append(hg); h["ga"].append(ag); h["tot"].append(tot); h["ov"].append(ov)
    h["hm_gf"].append(hg); h["hm_ga"].append(ag); h["ah_h"].append(ahw)
    a["gf"].append(ag); a["ga"].append(hg); a["tot"].append(tot); a["ov"].append(ov)
    a["ah_a"].append(1 - ahw)
    lg["tot"].append(tot); lg["ov"].append(ov); lg["ah_h"].append(ahw)


def find_optimal_threshold(probs, y_true, bm_dirs):
    """找最优决策阈值，同时决定是否用模型还是庄家"""
    best_acc = 0
    best_thr = 0.5
    best_method = "model"

    # 默认庄家准确率
    bm_acc = accuracy_score(y_true, bm_dirs)

    # 尝试不同阈值
    for thr in np.arange(0.35, 0.65, 0.01):
        preds = (probs >= thr).astype(int)
        acc = accuracy_score(y_true, preds)
        if acc > best_acc:
            best_acc = acc
            best_thr = thr

    # 如果模型即使最优阈值也不如庄家，用庄家
    if best_acc < bm_acc - 0.01:  # 1%容差
        return 0.5, "bookmaker", bm_acc

    return best_thr, "model", best_acc


def train_v5(code):
    print(f"\n{'='*60}\n{code} {LNAME.get(code,'')} V5\n{'='*60}")
    df = load_data(code, years=(2024, 2025))
    if len(df) < 150:
        print(f"  数据不足 {len(df)}，尝试2023-2025...")
        df = load_data(code, years=(2023, 2025))
    if df.empty or len(df) < 100:
        print("  无数据")
        return None

    n_total = len(df)
    n_pos = int(df["target"].sum())
    print(f"  {n_total}场, 上盘率 {n_pos/n_total:.1%}")

    X, y, info = build_features(df)
    if len(X) < 150:
        print(f"  样本不足 {len(X)}")
        return None

    sp = int(len(X) * 0.75)  # 用75%训练，25%测试
    Xtr, Xte = X.iloc[:sp], X.iloc[sp:]
    ytr, yte = y[:sp], y[sp:]
    itr, ite = info[:sp], info[sp:]

    bm_dirs_te = np.array([x["bm_dir"] for x in ite])
    bm_acc = accuracy_score(yte, bm_dirs_te)
    print(f"  训练{len(Xtr)} 测试{len(Xte)}, 庄家基准:{bm_acc:.1%}")

    sc = StandardScaler()
    Xs_tr = sc.fit_transform(Xtr)
    Xs_te = sc.transform(Xte)

    # 尝试多个配置，选最佳
    candidates = []

    if HAS_LGB:
        for lr in [0.03, 0.05, 0.08]:
            for md in [3, 5, 7]:
                for nl in [15, 31]:
                    clf = lgb_lib.LGBMClassifier(
                        n_estimators=300, learning_rate=lr, max_depth=md,
                        num_leaves=nl, min_child_samples=20,
                        subsample=0.8, colsample_bytree=0.7,
                        reg_alpha=0.5, reg_lambda=0.5,
                        random_state=42, verbosity=-1, n_jobs=-1
                    )
                    clf.fit(Xs_tr, ytr)
                    probs = clf.predict_proba(Xs_te)[:, 1]

                    # isotonic calibration
                    try:
                        cal = CalibratedClassifierCV(clf, cv=3, method="isotonic")
                        cal.fit(Xs_tr, ytr)
                        probs_cal = cal.predict_proba(Xs_te)[:, 1]
                    except:
                        probs_cal = probs

                    # 找最优阈值
                    for probs_used, label in [(probs, "raw"), (probs_cal, "cal")]:
                        best_thr, method, best_acc = find_optimal_threshold(probs_used, yte, bm_dirs_te)
                        candidates.append({
                            "clf": cal if label == "cal" else clf,
                            "acc": best_acc,
                            "thr": best_thr,
                            "method": method,
                            "label": f"LGB(lr={lr},d={md},nl={nl})-{label}",
                            "probs": probs_used,
                        })

    if HAS_XGB:
        for lr in [0.03, 0.05, 0.08]:
            for md in [3, 5, 7]:
                clf = xgb_lib.XGBClassifier(
                    n_estimators=300, learning_rate=lr, max_depth=md,
                    min_child_weight=20, subsample=0.8, colsample_bytree=0.7,
                    reg_alpha=0.5, reg_lambda=0.5,
                    random_state=42, verbosity=0, n_jobs=-1
                )
                clf.fit(Xs_tr, ytr)
                probs = clf.predict_proba(Xs_te)[:, 1]
                try:
                    cal = CalibratedClassifierCV(clf, cv=3, method="isotonic")
                    cal.fit(Xs_tr, ytr)
                    probs_cal = cal.predict_proba(Xs_te)[:, 1]
                except:
                    probs_cal = probs
                for probs_used, label in [(probs, "raw"), (probs_cal, "cal")]:
                    best_thr, method, best_acc = find_optimal_threshold(probs_used, yte, bm_dirs_te)
                    candidates.append({
                        "clf": cal if label == "cal" else clf,
                        "acc": best_acc,
                        "thr": best_thr,
                        "method": method,
                        "label": f"XGB(lr={lr},d={md})-{label}",
                        "probs": probs_used,
                    })

    # HGB
    for lr in [0.05, 0.08, 0.1]:
        for md in [3, 5, 7]:
            clf = HistGradientBoostingClassifier(
                max_iter=300, learning_rate=lr, max_depth=md,
                min_samples_leaf=20, l2_regularization=0.5,
                random_state=42
            )
            clf.fit(Xs_tr, ytr)
            probs = clf.predict_proba(Xs_te)[:, 1]
            best_thr, method, best_acc = find_optimal_threshold(probs, yte, bm_dirs_te)
            candidates.append({
                "clf": clf,
                "acc": best_acc,
                "thr": best_thr,
                "method": method,
                "label": f"HGB(lr={lr},d={md})",
                "probs": probs,
            })

    if not candidates:
        print("  无候选模型")
        return None

    candidates.sort(key=lambda x: -x["acc"])
    best = candidates[0]

    print(f"  最佳: {best['label']} thr={best['thr']:.2f} → {best['acc']:.1%} ({(best['acc']-bm_acc)*100:+.1f}pp) [{best['method']}]")

    # 策略分析
    probs_best = best["probs"]
    mod_dirs = (probs_best >= best["thr"]).astype(int)
    bm_dirs = bm_dirs_te

    print(f"\n  {'策略':<45} {'n':>4} {'准确率':>8}")
    print(f"  {'-'*60}")
    print(f"  {'基准-纯庄家方向':<45} {len(yte):>4} {bm_acc:>8.1%}")
    print(f"  {'模型最优阈值':<45} {len(yte):>4} {best['acc']:>8.1%}")

    for mc in [0.55, 0.60, 0.65, 0.70, 0.75, 0.80]:
        for agree in [False, True]:
            mask_model = ((probs_best >= mc) | (1 - probs_best >= mc))
            if agree:
                mask_model = mask_model & (bm_dirs == mod_dirs)
            else:
                mask_model = mask_model & (bm_dirs == (probs_best >= best["thr"]).astype(int))
            n = int(mask_model.sum())
            if n < 4:
                continue
            a = accuracy_score(yte[mask_model], mod_dirs[mask_model])
            lb = f"conf>{mc:.0%}" + ("+庄家一致" if agree else "")
            fl = " ✅" if a >= 0.85 else (" ⚠️" if a >= 0.70 else "")
            print(f"  {lb:<45} {n:>4} {a:>8.1%}{fl}")

    for mp in [0.85, 0.88, 0.90]:
        mask = ((probs_best >= mp) | (1 - probs_best >= mp))
        n = int(mask.sum())
        if n < 3:
            continue
        a = accuracy_score(yte[mask], mod_dirs[mask])
        fl = " ✅" if a >= 0.85 else (" ⚠️" if a >= 0.70 else "")
        print(f"  {'极端prob>'+f'{mp:.0%}':<45} {n:>4} {a:>8.1%}{fl}")

    # 保存
    joblib.dump(best["clf"], OUT / (code + "_model.pkl"))
    joblib.dump(sc, OUT / (code + "_scaler.pkl"))
    cfg = {
        "league": code,
        "league_name": LNAME.get(code, code),
        "accuracy": float(best["acc"]),
        "bookmaker_acc": float(bm_acc),
        "model_label": best["label"],
        "optimal_threshold": float(best["thr"]),
        "use_method": best["method"],
        "data_years": "2024-2025",
        "warmup": 8,
        "train_size": int(sp),
        "test_size": int(len(Xte)),
        "total_samples": int(n_total),
        "trained": datetime.now().isoformat(),
    }
    with open(OUT / (code + "_config.json"), "w") as f:
        json.dump(cfg, f, indent=2, ensure_ascii=False)
    print(f"  → {code}_model.pkl ({best['label']}, {best['acc']:.1%})")
    return cfg


def main():
    print("=" * 70)
    print("亚盘模型 v5 | 2024-2025数据 | 阈值优化 | 强正则化 | 校准")
    print("=" * 70)
    print(f"LightGBM: {'YES' if HAS_LGB else 'NO'}, XGBoost: {'YES' if HAS_XGB else 'NO'}")

    rs = {}
    for c in sorted(LNAME.keys()):
        r = train_v5(c)
        if r:
            rs[c] = r

    print("\n" + "=" * 70)
    print("汇总")
    print("=" * 70)
    print(f"\n{'联赛':<8} {'模型':<30} {'阈值':>6} {'方法':>10} {'准确率':>8} {'庄家':>8} {'提升':>8}")
    print("-" * 85)
    for c, cfg in sorted(rs.items(), key=lambda x: x[1]["accuracy"], reverse=True):
        lift = cfg["accuracy"] - cfg["bookmaker_acc"]
        s = "+" if lift > 0 else ""
        print(f"{cfg['league_name']:<8} {cfg['model_label']:<30} {cfg['optimal_threshold']:>6.2f} "
              f"{cfg['use_method']:>10} {cfg['accuracy']:.1%} {cfg['bookmaker_acc']:.1%} {s}{lift:.1%}")
    if rs:
        avg = float(np.mean([v["accuracy"] for v in rs.values()]))
        avg_b = float(np.mean([v["bookmaker_acc"] for v in rs.values()]))
        print("-" * 85)
        print(f"{'平均':<8} {'':<30} {'':>6} {'':>10} {avg:.1%} {avg_b:.1%} {'+' if avg > avg_b else ''}{(avg-avg_b)*100:.1f}pp")


if __name__ == "__main__":
    main()
