#!/usr/bin/env python3
"""
亚盘模型 V11 - CatBoost + 球队实体嵌入
- CatBoost原生处理球队名称（类别特征）无需编码
- 精简特征集减少过拟合
- 联赛组合策略：专注模型有稳定edge的联赛
"""
from __future__ import annotations
import os, sys, json, warnings
from collections import defaultdict
from pathlib import Path
from datetime import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score

warnings.filterwarnings('ignore')

try:
    import catboost as cb
    HAS_CB = True
except:
    HAS_CB = False

try:
    import lightgbm as lgb_lib
    HAS_LGB = True
except:
    HAS_LGB = False

try:
    import xgboost as xgb_lib
    HAS_XGB = True
except:
    HAS_XGB = False

REPO = Path("/root/.openclaw/workspace-flj_bot1/football-predictor-repro")
HIST = REPO / "historical_data"
OUT = REPO / "models/ah_models_v11"
OUT.mkdir(parents=True, exist_ok=True)

LNAME = {
    "e2":"英冠","e3":"英乙","e1":"英甲","d1":"德甲",
    "f1":"法甲","i1":"意甲","sp1":"西甲","sc1":"苏超",
    "tr1":"土超","p1":"葡超","e2n":"荷乙","cz1":"捷克甲",
}


def pr(odds):
    try:
        v = float(odds)
        return 1.0 / v if v > 1 else np.nan
    except:
        return np.nan


def avg5(v):
    if not v: return 2.5
    v = list(v)
    return float(np.mean(v[-5:] if len(v) >= 5 else v))


def wr(v):
    if not v: return 0.5
    return float(np.mean(list(v)))


def strk(v, t):
    s = 0
    for x in reversed(list(v) if v else []):
        if x == t: s += 1
        else: break
    return s


def load_data(code):
    hp = HIST / (code + "_history.csv")
    if not hp.exists():
        return pd.DataFrame()
    df = pd.read_csv(hp, low_memory=False)
    df["Date"] = pd.to_datetime(df["Date"], errors="coerce", dayfirst=True)
    df = df[(df["Date"] >= "2023-06-01") & (df["Date"] <= "2025-12-31")]
    df = df.dropna(subset=["FTHG", "FTAG", "AHh", "B365AHH", "B365AHA"])
    df = df[df["B365AHH"].astype(float) > 1].copy()
    df = df.sort_values("Date").reset_index(drop=True)
    df["gd"] = df["FTHG"].astype(float) - df["FTAG"].astype(float)
    df["gd_ah"] = df["gd"] + df["AHh"].astype(float)
    df = df[df["gd_ah"].abs() >= 0.01].reset_index(drop=True)
    df["target"] = (df["gd_ah"] > 0).astype(int)
    bm_ph = df["B365AHH"].apply(pr)
    bm_pa = df["B365AHA"].apply(pr)
    ov = bm_ph + bm_pa
    df["bm_ph_n"] = bm_ph / ov
    df["bm_pa_n"] = bm_pa / ov
    df["bm_dir"] = (df["bm_ph_n"] > 0.5).astype(int)
    df["bm_conf"] = (df["bm_ph_n"] - 0.5).abs() * 2
    df["clv"] = (df["bm_ph_n"] - 0.5).abs()
    return df


def build_features(df, ts, lg):
    """精简特征集 + 球队名称（类别特征）"""
    row = df
    hm = str(row["HomeTeam"])
    aw = str(row["AwayTeam"])
    hg = float(row["FTHG"])
    ag = float(row["FTAG"])
    ahh = float(row["AHh"])
    bm_ph = float(row["bm_ph_n"])
    bm_pa = float(row["bm_pa_n"])
    bm_dir = int(row["bm_dir"])
    clv = float(row.get("clv", 0))
    h = ts.get(hm, {})
    a = ts.get(aw, {})
    lg_ah = wr(lg["ah_h"][-50:]) if lg["ah_h"] else 0.5
    ht_ah10 = wr(h.get("ah_h", [])[-10:] if h.get("ah_h") else [])
    at_ah10 = wr(a.get("ah_a", [])[-10:] if a.get("ah_a") else [])
    ht_gf = avg5(h.get("gf", []))
    at_gf = avg5(a.get("gf", []))
    ht_ga = avg5(h.get("ga", []))
    at_ga = avg5(a.get("ga", []))
    eg_diff = ht_gf - at_ga

    f = {}
    # 球队名称（类别）
    f["home_team"] = hm
    f["away_team"] = aw
    # 核心数值特征（精简）
    f["bm_edge"] = bm_ph - 0.5
    f["bm_conf"] = abs(bm_ph - 0.5) * 2
    f["bm_dir"] = bm_dir
    f["clv"] = clv
    f["ahh"] = ahh
    f["ahh_abs"] = abs(ahh)
    f["bm_vs_ht10"] = float(bm_dir) - ht_ah10
    f["bm_vs_at10"] = float(1 - bm_dir) - at_ah10
    f["bm_vs_lg"] = bm_ph - lg_ah
    f["ht_ah10"] = ht_ah10
    f["at_ah10"] = at_ah10
    f["ht_ah_strk"] = strk(h.get("ah_h", []), 1)
    f["at_ah_strk"] = strk(a.get("ah_a", []), 1)
    f["ht_gf_avg"] = ht_gf
    f["at_gf_avg"] = at_gf
    f["ht_ga_avg"] = ht_ga
    f["at_ga_avg"] = at_ga
    f["ht_tot_avg"] = avg5(h.get("tot", []))
    f["at_tot_avg"] = avg5(a.get("tot", []))
    f["ht_ov10"] = wr(h.get("ov", [])[-10:] if h.get("ov") else [])
    f["at_ov10"] = wr(a.get("ov", [])[-10:] if a.get("ov") else [])
    f["eg_diff"] = eg_diff
    f["att_diff"] = ht_gf - at_gf
    f["def_diff"] = at_ga - ht_ga
    f["lg_ah"] = lg_ah
    f["form_diff"] = wr(h.get("ov", [])[-5:]) - wr(a.get("ov", [])[-5:]) if (h.get("ov") and a.get("ov")) else 0
    f["ht_ah_L3"] = float(sum(h.get("ah_h", [])[-3:]) if len(h.get("ah_h", [])) >= 3 else 0)
    f["at_ah_L3"] = float(sum(a.get("ah_a", [])[-3:]) if len(a.get("ah_a", [])) >= 3 else 0)
    f["ht_ah_L5"] = float(sum(h.get("ah_h", [])[-5:]) if len(h.get("ah_h", [])) >= 5 else 0)
    f["at_ah_L5"] = float(sum(a.get("ah_a", [])[-5:]) if len(a.get("ah_a", [])) >= 5 else 0)
    f["ht_ah10_raw"] = float(ht_ah10)
    f["at_ah10_raw"] = float(at_ah10)
    f["lg_ah_raw"] = float(lg_ah)
    f["bm_conf_raw"] = float(abs(bm_ph - 0.5) * 2)

    # Update
    tot = hg + ag
    ovv = int(tot > 2.5)
    ahw = int(hg - ag + ahh > 0)
    h2 = ts[hm]
    a2 = ts[aw]
    h2["gf"].append(hg); h2["ga"].append(ag); h2["tot"].append(tot); h2["ov"].append(ovv)
    h2["ah_h"].append(ahw)
    a2["gf"].append(ag); a2["ga"].append(hg); a2["tot"].append(tot); a2["ov"].append(ovv)
    a2["ah_a"].append(1 - ahw)
    lg["tot"].append(tot); lg["ov"].append(ovv); lg["ah_h"].append(ahw)
    return f


def _upd(ts, lg, row):
    hm = str(row["HomeTeam"])
    aw = str(row["AwayTeam"])
    hg = float(row["FTHG"])
    ag = float(row["FTAG"])
    ahh = float(row["AHh"])
    tot = hg + ag
    ovv = int(tot > 2.5)
    ahw = int(hg - ag + ahh > 0)
    h = ts[hm]
    a = ts[aw]
    h["gf"].append(hg); h["ga"].append(ag); h["tot"].append(tot); h["ov"].append(ovv)
    h["ah_h"].append(ahw)
    a["gf"].append(ag); a["ga"].append(hg); a["tot"].append(tot); a["ov"].append(ovv)
    a["ah_a"].append(1 - ahw)
    lg["tot"].append(tot); lg["ov"].append(ovv); lg["ah_h"].append(ahw)


# 数值特征
NUM_FEATURES = [
    "bm_edge", "bm_conf", "bm_dir", "clv",
    "ahh", "ahh_abs",
    "bm_vs_ht10", "bm_vs_at10", "bm_vs_lg",
    "ht_ah10", "at_ah10",
    "ht_ah_strk", "at_ah_strk",
    "ht_gf_avg", "at_gf_avg", "ht_ga_avg", "at_ga_avg",
    "ht_tot_avg", "at_tot_avg",
    "ht_ov10", "at_ov10",
    "eg_diff", "att_diff", "def_diff",
    "lg_ah", "form_diff",
    "ht_ah_L3", "at_ah_L3", "ht_ah_L5", "at_ah_L5",
]

CAT_FEATURES = ["home_team", "away_team"]


def run_v11(code):
    print(f"\n{'='*65}\n{code} {LNAME.get(code,'')} V11 CatBoost\n{'='*65}")
    df = load_data(code)
    if df.empty:
        print(" 无数据"); return None
    total = len(df)
    print(f" {total}场, 上盘率 {df['target'].mean():.1%}")

    n = len(df)
    min_train = 300
    n_windows = 5
    window_size = max(50, (n - min_train) // n_windows)
    ew_results = []

    for w in range(n_windows):
        train_end = min_train + w * window_size
        test_start = train_end
        test_end = min(test_start + window_size, n)
        if test_end - test_start < 30:
            continue

        df_tr = df.iloc[:train_end]
        df_te = df.iloc[test_start:test_end]

        # Build features
        ts = defaultdict(lambda: {"gf": [], "ga": [], "tot": [], "ov": [], "ah_h": [], "ah_a": []})
        lg = {"tot": [], "ov": [], "ah_h": []}
        Xl_tr, yl_tr = [], []
        for i, row in df_tr.iterrows():
            if i < 8: _upd(ts, lg, row); continue
            f = build_features(row, ts, lg)
            Xl_tr.append(f); yl_tr.append(int(row["target"]))

        ts_te = defaultdict(lambda: {"gf": [], "ga": [], "tot": [], "ov": [], "ah_h": [], "ah_a": []})
        lg_te = {"tot": [], "ov": [], "ah_h": []}
        for i, row in df_tr.iterrows():
            _upd(ts_te, lg_te, row)
        Xl_te, yl_te, info_te = [], [], []
        for i, row in df_te.iterrows():
            f = build_features(row, ts_te, lg_te)
            Xl_te.append(f); yl_te.append(int(row["target"]))
            info_te.append({"bm_dir": int(row["bm_dir"])})

        Xtr_df = pd.DataFrame(Xl_tr)
        ytr = np.array(yl_tr)
        Xte_df = pd.DataFrame(Xl_te)
        yte = np.array(yl_te)
        bm_dirs_te = np.array([x["bm_dir"] for x in info_te])

        if len(Xte_df) < 20:
            continue

        bm_acc = accuracy_score(yte, bm_dirs_te)

        # Train CatBoost
        if HAS_CB:
            pool_tr = cb.Pool(Xtr_df, ytr, cat_features=CAT_FEATURES)
            pool_te = cb.Pool(Xte_df, cat_features=CAT_FEATURES)

            cb_model = cb.CatBoostClassifier(
                iterations=300,
                learning_rate=0.05,
                depth=6,
                l2_leaf_reg=3,
                random_seed=42,
                verbose=0,
                task_type='CPU'
            )
            cb_model.fit(pool_tr)
            cb_probs = cb_model.predict_proba(pool_te)[:, 1]

            # Also train LGB
            Xtr_num = Xtr_df[NUM_FEATURES]
            Xte_num = Xte_df[NUM_FEATURES]
            sc = StandardScaler()
            Xs_tr = sc.fit_transform(Xtr_num)
            Xs_te = sc.transform(Xte_num)

            lgb_probs = None
            if HAS_LGB:
                lgb_model = lgb_lib.LGBMClassifier(
                    n_estimators=200, learning_rate=0.05, max_depth=5,
                    num_leaves=31, min_child_samples=20,
                    subsample=0.8, colsample_bytree=0.8,
                    reg_alpha=0.3, reg_lambda=0.3,
                    random_state=42, verbosity=-1, n_jobs=-1
                )
                lgb_model.fit(Xs_tr, ytr)
                lgb_probs = lgb_model.predict_proba(Xs_te)[:, 1]

            # Ensemble: CB + LGB (if available)
            if lgb_probs is not None:
                probs = 0.6 * cb_probs + 0.4 * lgb_probs
            else:
                probs = cb_probs
        else:
            # Fallback to LGB only
            Xtr_num = Xtr_df[NUM_FEATURES]
            Xte_num = Xte_df[NUM_FEATURES]
            sc = StandardScaler()
            Xs_tr = sc.fit_transform(Xtr_num)
            Xs_te = sc.transform(Xte_num)
            lgb_model = lgb_lib.LGBMClassifier(
                n_estimators=200, learning_rate=0.05, max_depth=5,
                num_leaves=31, min_child_samples=20,
                random_state=42, verbosity=-1, n_jobs=-1
            )
            lgb_model.fit(Xs_tr, ytr)
            probs = lgb_model.predict_proba(Xs_te)[:, 1]

        # Optimal threshold
        best_acc = 0; best_thr = 0.5
        for thr in np.arange(0.35, 0.70, 0.02):
            acc = accuracy_score(yte, (probs >= thr).astype(int))
            if acc > best_acc:
                best_acc = acc; best_thr = thr

        ew_results.append({
            "window": w + 1, "train_n": len(Xtr_df), "test_n": len(Xte_df),
            "bm_acc": bm_acc, "model_acc": best_acc, "optimal_thr": best_thr,
        })

    if ew_results:
        print(f"\n  Expanding Window ({len(ew_results)} folds):")
        for r in ew_results:
            diff = r["model_acc"] - r["bm_acc"]
            s = "+" if diff > 0 else ""
            print(f"    W{r['window']}: train={r['train_n']} test={r['test_n']} "
                  f"庄家={r['bm_acc']:.1%} 模型={r['model_acc']:.1%} {s}{diff:.1%}pp thr={r['optimal_thr']:.2f}")
        avg_bm = float(np.mean([r["bm_acc"] for r in ew_results]))
        avg_mod = float(np.mean([r["model_acc"] for r in ew_results]))
        avg_thr = float(np.mean([r["optimal_thr"] for r in ew_results]))
        print(f"  平均: 庄家{avg_bm:.1%} 模型{avg_mod:.1%} {'+' if avg_mod > avg_bm else ''}{(avg_mod-avg_bm)*100:.1f}pp avg_thr={avg_thr:.2f}")

    # Final
    sp = int(total * 0.80)
    df_tr = df.iloc[:sp]
    df_te = df.iloc[sp:]
    if len(df_te) < 50:
        return None

    ts = defaultdict(lambda: {"gf": [], "ga": [], "tot": [], "ov": [], "ah_h": [], "ah_a": []})
    lg = {"tot": [], "ov": [], "ah_h": []}
    Xl_tr, yl_tr = [], []
    for i, row in df_tr.iterrows():
        if i < 8: _upd(ts, lg, row); continue
        f = build_features(row, ts, lg)
        Xl_tr.append(f); yl_tr.append(int(row["target"]))

    ts_te = defaultdict(lambda: {"gf": [], "ga": [], "tot": [], "ov": [], "ah_h": [], "ah_a": []})
    lg_te = {"tot": [], "ov": [], "ah_h": []}
    for i, row in df_tr.iterrows():
        _upd(ts_te, lg_te, row)
    Xl_te, yl_te, info_te = [], [], []
    for i, row in df_te.iterrows():
        f = build_features(row, ts_te, lg_te)
        Xl_te.append(f); yl_te.append(int(row["target"]))
        info_te.append({"bm_dir": int(row["bm_dir"]), "clv": float(row.get("clv", 0))})

    Xtr_df = pd.DataFrame(Xl_tr)
    ytr = np.array(yl_tr)
    Xte_df = pd.DataFrame(Xl_te)
    yte = np.array(yl_te)
    bm_dirs_te = np.array([x["bm_dir"] for x in info_te])
    clv_te = np.array([x["clv"] for x in info_te])

    bm_acc = accuracy_score(yte, bm_dirs_te)

    if HAS_CB:
        pool_tr = cb.Pool(Xtr_df, ytr, cat_features=CAT_FEATURES)
        pool_te = cb.Pool(Xte_df, cat_features=CAT_FEATURES)
        cb_model = cb.CatBoostClassifier(
            iterations=300, learning_rate=0.05, depth=6,
            l2_leaf_reg=3, random_seed=42, verbose=0
        )
        cb_model.fit(pool_tr)
        cb_probs = cb_model.predict_proba(pool_te)[:, 1]

        Xtr_num = Xtr_df[NUM_FEATURES]
        Xte_num = Xte_df[NUM_FEATURES]
        sc = StandardScaler()
        Xs_tr = sc.fit_transform(Xtr_num)
        Xs_te = sc.transform(Xte_num)

        lgb_probs = None
        if HAS_LGB:
            lgb_model = lgb_lib.LGBMClassifier(
                n_estimators=200, learning_rate=0.05, max_depth=5,
                num_leaves=31, min_child_samples=20,
                subsample=0.8, colsample_bytree=0.8,
                reg_alpha=0.3, reg_lambda=0.3,
                random_state=42, verbosity=-1, n_jobs=-1
            )
            lgb_model.fit(Xs_tr, ytr)
            lgb_probs = lgb_model.predict_proba(Xs_te)[:, 1]

        if lgb_probs is not None:
            probs = 0.6 * cb_probs + 0.4 * lgb_probs
        else:
            probs = cb_probs
    else:
        Xtr_num = Xtr_df[NUM_FEATURES]
        Xte_num = Xte_df[NUM_FEATURES]
        sc = StandardScaler()
        Xs_tr = sc.fit_transform(Xtr_num)
        Xs_te = sc.transform(Xte_num)
        lgb_model = lgb_lib.LGBMClassifier(
            n_estimators=200, learning_rate=0.05, max_depth=5,
            num_leaves=31, min_child_samples=20,
            random_state=42, verbosity=-1, n_jobs=-1
        )
        lgb_model.fit(Xs_tr, ytr)
        probs = lgb_model.predict_proba(Xs_te)[:, 1]

    best_acc = 0; best_thr = 0.5
    for thr in np.arange(0.35, 0.70, 0.01):
        acc = accuracy_score(yte, (probs >= thr).astype(int))
        if acc > best_acc:
            best_acc = acc; best_thr = thr

    mod_dirs = (probs >= best_thr).astype(int)

    print(f"\n  最终测试 (n={len(yte)}, thr={best_thr:.2f}):")
    print(f"  {'庄家方向':<45} {len(yte):>4} {bm_acc:>8.1%}")
    print(f"  {'CB+LGB Ensemble':<45} {len(yte):>4} {best_acc:>8.1%}")

    # Strategies
    for conf in [0.60, 0.65, 0.70, 0.75, 0.80]:
        mask = ((probs >= conf) | (1 - probs >= conf)) & (mod_dirs == bm_dirs_te)
        n_m = int(mask.sum())
        if n_m < 4: continue
        a = accuracy_score(yte[mask], mod_dirs[mask])
        flag = " ✅" if a >= 0.85 else (" ⚠️" if a >= 0.75 else "")
        print(f"  conf>={conf:.0%}+一致: n={n_m:>3} acc={a:.1%}{flag}")

    tier_green = ((probs >= 0.75) | (1 - probs >= 0.75)) & (mod_dirs == bm_dirs_te)
    n_g = int(tier_green.sum())
    acc_g = accuracy_score(yte[tier_green], mod_dirs[tier_green]) if n_g > 0 else 0
    print(f"  绿色(conf>=75%+一致): {n_g}场 {acc_g:.1%} {'✅ 重点投' if acc_g >= 0.75 else '⚠️'}")

    avg_ew_bm = float(np.mean([r["bm_acc"] for r in ew_results])) if ew_results else bm_acc
    avg_ew_mod = float(np.mean([r["model_acc"] for r in ew_results])) if ew_results else best_acc

    cfg = {
        "league": code,
        "league_name": LNAME.get(code, code),
        "ew_avg_bm": avg_ew_bm,
        "ew_avg_model": avg_ew_mod,
        "final_bm_acc": float(bm_acc),
        "final_model_acc": float(best_acc),
        "final_optimal_thr": float(best_thr),
        "green_n": n_g,
        "green_acc": float(acc_g),
        "total_n": int(total),
        "trained": datetime.now().isoformat(),
    }
    with open(OUT / (code + "_config.json"), "w") as f:
        json.dump(cfg, f, indent=2, ensure_ascii=False)
    print(f"  → EW: {avg_ew_mod:.1%}, 最终: {best_acc:.1%}")
    return cfg


def main():
    print("=" * 70)
    print("亚盘模型 V11 | CatBoost球队嵌入 | CB+LGB Ensemble")
    print("=" * 70)
    rs = {}
    for c in sorted(LNAME.keys()):
        r = run_v11(c)
        if r:
            rs[c] = r

    print("\n" + "=" * 70)
    print("V11 最终汇总")
    print("=" * 70)
    print(f"\n{'联赛':<8} {'EW庄家':>8} {'EW模型':>8} {'提升':>8} {'最终':>8} {'绿色n':>6} {'绿色acc':>8}")
    print("-" * 70)
    for c, cfg in sorted(rs.items(), key=lambda x: x[1]["ew_avg_model"], reverse=True):
        ew_lift = cfg["ew_avg_model"] - cfg["ew_avg_bm"]
        s = "+" if ew_lift > 0 else ""
        print(f"{cfg['league_name']:<8} {cfg['ew_avg_bm']:>8.1%} {cfg['ew_avg_model']:>8.1%} "
              f"{s}{ew_lift*100:>7.1f}pp {cfg['final_model_acc']:>8.1%} "
              f"{cfg['green_n']:>6} {cfg['green_acc']:>8.1%}")
    if rs:
        avg_ew = float(np.mean([v["ew_avg_model"] for v in rs.values()]))
        avg_bm = float(np.mean([v["ew_avg_bm"] for v in rs.values()]))
        print("-" * 70)
        print(f"{'EW平均':<8} {avg_bm:>8.1%} {avg_ew:>8.1%} "
              f"{'+' if avg_ew > avg_bm else ''}{(avg_ew-avg_bm)*100:.1f}pp")

    print("\n" + "=" * 70)
    print("绿色层汇总 (目标75%+)")
    print("=" * 70)
    for c, cfg in sorted(rs.items(), key=lambda x: -x[1]["green_acc"] if x[1]["green_n"] >= 5 else 0):
        if cfg["green_n"] < 5: continue
        flag = " ✅" if cfg["green_acc"] >= 0.75 else (" ⚠️" if cfg["green_acc"] >= 0.70 else "")
        print(f"  {cfg['league_name']:<8} n={cfg['green_n']:>3} acc={cfg['green_acc']:.1%}{flag}")


if __name__ == "__main__":
    main()
