#!/usr/bin/env python3
"""
赔率驱动 + 市场偏离 检测 大小球预测 v3
=======================================
核心策略：只下"市场偏离"注
- 赔率隐含概率是当前市场共识（已包含球队状态、伤停、战意等所有信息）
- 模型的作用是判断：市场赔率是否高估/低估了大球概率
- 投注方向：模型概率 vs 市场概率有显著偏离时，跟随模型

关键改进：
1. 只用最近2个赛季（避免历史分布偏差）
2. 市场偏离阈值（不用绝对概率阈值）
3. 只下注模型与市场显著不一致的场次
"""
import json, os, sys, time, warnings
from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score

warnings.filterwarnings('ignore')

REPO_ROOT = Path(__file__).resolve().parents[1]
HIST_DIR  = REPO_ROOT / 'historical_data'
OUTPUT_DIR = REPO_ROOT / 'experiments' / 'goal_number_mar_april_v3'
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

# ===== 超参 =====
RECENT_SEASONS = 2        # 只用最近2赛季
EDGE_THRESH    = 0.07     # 模型概率 vs 市场概率，偏离>7%才下注
MIN_BET        = 8         # 窗口最少投注数
WF_TRAIN       = 200       # Walk-Forward训练窗口
WF_TEST        = 30        # Walk-Forward测试窗口
WF_STEP        = 15

LEAGUES = {
    'd1':  {'name': '德甲',  'has_odds': True},
    'sp1': {'name': '西甲',  'has_odds': True},
    'i1':  {'name': '意甲',  'has_odds': True},
    'e0':  {'name': '英超',  'has_odds': False},
    'f2':  {'name': '法乙',  'has_odds': False},
    'r1':  {'name': '俄超',  'has_odds': False},
    'g1':  {'name': '希腊超', 'has_odds': False},
}


def load_league(code):
    csv_path = HIST_DIR / f'{code}_history.csv'
    if not csv_path.exists():
        return None
    df = pd.read_csv(csv_path, low_memory=False)
    df['Date'] = pd.to_datetime(df['Date'], dayfirst=True, errors='coerce')
    df = df.sort_values('Date').reset_index(drop=True)
    df['FTHG'] = pd.to_numeric(df['FTHG'], errors='coerce')
    df['FTAG'] = pd.to_numeric(df['FTAG'], errors='coerce')
    df['tot'] = df['FTHG'] + df['FTAG']
    df['is_over'] = (df['tot'] > 2.5).astype(int)

    max_year = df['Date'].max().year
    df = df[df['Date'].dt.year >= max_year - RECENT_SEASONS].copy()

    if LEAGUES[code]['has_odds']:
        for oo, uu in [('B365>2.5','B365<2.5'), ('Avg>2.5','Avg<2.5')]:
            if oo in df.columns:
                df['over_odds'] = pd.to_numeric(df[oo], errors='coerce')
                df['under_odds'] = pd.to_numeric(df[uu], errors='coerce')
                df['mkt_over'] = df['over_odds'].apply(lambda x: 1/x if x>1 else np.nan)
                df['mkt_und']  = df['under_odds'].apply(lambda x: 1/x if x>1 else np.nan)
                break
    return df


def build_features(df):
    """只用近因窗口特征 + 赔率特征"""
    th = defaultdict(lambda: {
        'gf3':[],'ga3':[],'over3':[],
        'gf5':[],'ga5':[],'over5':[],
        'gf10':[],'ga10':[],'over10':[],
        'hgf':[],'hga':[],'hovr':[],
        'agf':[],'aga':[],'aovr':[],
    })

    rows = []
    for _, row in df.iterrows():
        home = str(row.get('HomeTeam',''))
        away = str(row.get('AwayTeam',''))
        h = th[home]; a = th[away]

        def w3(v):  return float(np.mean(v[-3:]))  if len(v)>=3 else (float(np.mean(v)) if v else np.nan)
        def w5(v):  return float(np.mean(v[-5:]))  if len(v)>=5 else (float(np.mean(v)) if v else np.nan)
        def w10(v): return float(np.mean(v[-10:])) if len(v)>=10 else (float(np.mean(v)) if v else np.nan)
        def strk(v,val=1):
            s=0
            for x in reversed(v):
                if x==val: s+=1
                else: break
            return s

        r = {
            'home_gf3': w3(h['gf3']), 'home_ga3': w3(h['ga3']),
            'away_gf3': w3(a['gf3']), 'away_ga3': w3(a['ga3']),
            'home_gf5': w5(h['gf5']), 'home_ga5': w5(h['ga5']),
            'away_gf5': w5(a['gf5']), 'away_ga5': w5(a['ga5']),
            'home_gf10':w10(h['gf10']),'home_ga10':w10(h['ga10']),
            'away_gf10':w10(a['gf10']),'away_ga10':w10(a['ga10']),
            'home_ovr3': w3(h['over3']), 'away_ovr3': w3(a['over3']),
            'home_ovr5': w5(h['over5']), 'away_ovr5': w5(a['over5']),
            'home_ovr10':w10(h['over10']),'away_ovr10':w10(a['over10']),
            'home_ovr_strk': strk(h['over3'],1), 'home_und_strk': strk(h['over3'],0),
            'away_ovr_strk': strk(a['over3'],1), 'away_und_strk': strk(a['over3'],0),
            'home_hgf': float(np.mean(h['hgf'])) if h['hgf'] else np.nan,
            'home_hga': float(np.mean(h['hga'])) if h['hga'] else np.nan,
            'away_agf': float(np.mean(a['agf'])) if a['agf'] else np.nan,
            'away_aga': float(np.mean(a['aga'])) if a['aga'] else np.nan,
            'exp_g_home': w5(h['gf5']) + w5(a['aga']),
            'exp_g_away': w5(a['gf5']) + w5(h['hga']),
            'exp_tot':    (w5(h['gf5'])+w5(h['ga5'])+w5(a['gf5'])+w5(a['ga5']))/2,
            'recent_ovr_diff': w5(h['over5']) - w5(a['over5']),
            'home_gd3':  w3(h['gf3'])-w3(h['ga3']),
            'away_gd3':  w3(a['gf3'])-w3(a['ga3']),
        }

        # 赔率特征（核心！用market fair probability而非原始odds）
        if pd.notna(row.get('mkt_over')) and pd.notna(row.get('mkt_und')):
            mko = float(row['mkt_over'])
            mku = float(row['mkt_und'])
            fair = mko / (mko + mku)
            r['mkt_over']   = mko
            r['mkt_fair']   = fair
            r['mkt_margin'] = mko + mku - 1
        else:
            r['mkt_over'] = r['mkt_fair'] = 0.5
            r['mkt_margin'] = 0

        r['is_over'] = int(row['is_over'])
        rows.append(r)

        # 更新
        tot = float(row['tot'])
        io  = int(row['is_over'])
        hg  = float(row['FTHG']) if pd.notna(row.get('FTHG')) else tot
        ag  = float(row['FTAG']) if pd.notna(row.get('FTAG')) else tot

        for s in [h, a]:
            s['gf3'].append(tot); s['ga3'].append(tot); s['over3'].append(io)
            s['gf5'].append(tot); s['ga5'].append(tot); s['over5'].append(io)
            s['gf10'].append(tot);s['ga10'].append(tot);s['over10'].append(io)

        if pd.notna(row.get('FTHG')):
            h['hgf'].append(float(row['FTHG']))
            h['hga'].append(float(row['FTAG']))
            h['hovr'].append(io)
            a['agf'].append(float(row['FTAG']))
            a['aga'].append(float(row['FTHG']))
            a['aovr'].append(io)

    return pd.DataFrame(rows)


def walk_forward_market_edge(X, y, model_fn, edge=EDGE_THRESH):
    """
    Walk-Forward with market-edge betting:
    - 投注"大球"当: model_prob >= market_fair + edge
    - 投注"小球"当: model_prob <= market_fair - edge
    - 其他: 跳过
    """
    n = len(X)
    if n < WF_TRAIN + WF_TEST:
        return None

    mkt_col = 'mkt_fair'
    if mkt_col not in X.columns:
        return None

    all_true=[]; all_pred=[]; all_bet=[]

    for start in range(0, n-WF_TRAIN-WF_TEST+1, WF_STEP):
        tr_end=start+WF_TRAIN; te_end=tr_end+WF_TEST
        X_tr,X_te = X.iloc[start:tr_end], X.iloc[tr_end:te_end]
        y_tr,y_te = y.iloc[start:tr_end], y.iloc[tr_end:te_end]
        if len(X_tr)<80 or len(X_te)<5: continue

        scaler = StandardScaler()
        X_tr_s = scaler.fit_transform(X_tr.fillna(X_tr.median()))
        X_te_s = scaler.transform(X_te.fillna(X_tr.median()))

        model = model_fn()
        model.fit(X_tr_s, y_tr)
        probs = model.predict_proba(X_te_s)[:,1]
        mkt   = X_te[mkt_col].values

        # 市场偏离投注
        preds = np.full(len(probs), -1, dtype=int)  # -1=skip
        preds[(probs - mkt) >= edge] = 1   # over
        preds[(mkt - probs) >= edge] = 0   # under
        bet_mask = preds != -1

        all_true.extend(y_te.values)
        all_pred.extend(preds)
        all_bet.extend(bet_mask)

    if not all_true:
        return None

    all_true = np.array(all_true)
    all_pred = np.array(all_pred)
    all_bet  = np.array(all_bet)

    overall = accuracy_score(all_true, (all_pred != -1).astype(int) *
                              (all_pred == 1).astype(int))

    bet_t = all_true[all_bet]
    bet_p = all_pred[all_bet]
    bet_acc = accuracy_score(bet_t, bet_p) if all_bet.sum() >= MIN_BET else 0.0

    return {
        'overall': float(overall),
        'bet_acc': float(bet_acc),
        'bet_count': int(all_bet.sum()),
        'edge': edge,
    }


def walk_forward_threshold(X, y, model_fn, t_high=0.65, t_low=0.35):
    """Walk-Forward with absolute threshold"""
    n = len(X)
    if n < WF_TRAIN + WF_TEST:
        return None

    all_true=[]; all_prob=[]; all_bet=[]

    for start in range(0, n-WF_TRAIN-WF_TEST+1, WF_STEP):
        tr_end=start+WF_TRAIN; te_end=tr_end+WF_TEST
        X_tr,X_te = X.iloc[start:tr_end], X.iloc[tr_end:te_end]
        y_tr,y_te = y.iloc[start:tr_end], y.iloc[tr_end:te_end]
        if len(X_tr)<80 or len(X_te)<5: continue

        scaler = StandardScaler()
        X_tr_s = scaler.fit_transform(X_tr.fillna(X_tr.median()))
        X_te_s = scaler.transform(X_te.fillna(X_tr.median()))

        model = model_fn()
        model.fit(X_tr_s, y_tr)
        probs = model.predict_proba(X_te_s)[:,1]
        preds = np.zeros(len(probs), dtype=int)
        preds[probs>=t_high]=1
        preds[probs<=t_low]=0
        bet_mask = (probs>=t_high)|(probs<=t_low)

        all_true.extend(y_te.values)
        all_prob.extend(probs)
        all_bet.extend(bet_mask)

    if not all_true:
        return None

    all_true=np.array(all_true); all_prob=np.array(all_prob); all_bet=np.array(all_bet)
    overall=accuracy_score(all_true,(all_prob>0.5).astype(int))
    bet_t=all_true[all_bet]; bet_p=(all_prob[all_bet]>=t_high).astype(int)
    bet_acc=accuracy_score(bet_t,bet_p) if all_bet.sum()>=MIN_BET else overall
    return {'overall':float(overall),'bet_acc':float(bet_acc),'bet_count':int(all_bet.sum())}


def train_league(code):
    print(f"\n{'='*60}")
    print(f"  {LEAGUES[code]['name']} ({code}) - v3 市场偏离策略")
    print(f"{'='*60}")

    df = load_league(code)
    if df is None: print("  [SKIP]"); return None

    has_odds = LEAGUES[code]['has_odds']
    print(f"  数据: {len(df)}场, 大球率:{df['is_over'].mean():.1%}")

    if has_odds:
        df_odds = df[df['mkt_over'].notna() & (df['mkt_over']>0)].copy()
    else:
        df_odds = df.copy()
    print(f"  有赔率: {len(df_odds)}场")

    feat_df = build_features(df_odds)
    cols = [c for c in feat_df.columns if c!='is_over']
    nan = feat_df[cols].isna().mean()
    valid = [c for c in cols if nan[c]<0.5]
    X = feat_df[valid].fillna(0)
    y = feat_df['is_over'].astype(int)

    df_odds = df_odds.reset_index(drop=True)
    X = X.reset_index(drop=True)
    y = y.reset_index(drop=True)

    mar_mask = (df_odds['Date'].dt.month.isin([3,4]))&(df_odds['Date'].dt.year==2026)
    mar_idx = df_odds[mar_mask].index.tolist()
    train_idx = [i for i in df_odds.index if i not in mar_idx]

    X_train = X.iloc[train_idx]; y_train = y.iloc[train_idx]
    X_mar = X.iloc[mar_idx]; y_mar = y.iloc[mar_idx].values

    print(f"  特征:{len(valid)} | 训练:{len(X_train)} | 测试Mar-Apr:{len(X_mar)}")

    models = [
        ('RF_d5', lambda: RandomForestClassifier(
            n_estimators=200,max_depth=5,min_samples_leaf=15,
            max_features=0.5,random_state=42,n_jobs=-1)),
        ('RF_d4', lambda: RandomForestClassifier(
            n_estimators=150,max_depth=4,min_samples_leaf=20,
            max_features=0.4,random_state=42,n_jobs=-1)),
        ('GB', lambda: GradientBoostingClassifier(
            n_estimators=100,max_depth=3,learning_rate=0.05,
            min_samples_leaf=25,subsample=0.8,random_state=42)),
        ('LR', lambda: LogisticRegression(C=0.05,max_iter=1000,random_state=42)),
    ]

    best=None; best_name=None; best_fn=None

    # 市场偏离Walk-Forward
    print(f"\n  策略A: 市场偏离(edge={EDGE_THRESH})")
    for name, fn in models:
        r = walk_forward_market_edge(X_train, y_train, fn)
        if r:
            print(f"    {name}: 投注={r['bet_acc']:.1%}(n={r['bet_count']})")
            if best is None or r['bet_acc']>best['bet_acc']:
                best=r; best_name=name; best_fn=fn

    # 绝对阈值Walk-Forward
    print(f"\n  策略B: 绝对阈值(0.65/0.35)")
    for name, fn in models:
        r = walk_forward_threshold(X_train, y_train, fn)
        if r:
            print(f"    {name}: 投注={r['bet_acc']:.1%}(n={r['bet_count']})")
            if best is None or r['bet_acc']>best['bet_acc']:
                best=r; best_name=name; best_fn=fn

    if best is None:
        print("  [FAIL]"); return None

    print(f"\n  最佳: {best_name} 投注={best['bet_acc']:.1%}(n={best['bet_count']})")

    # 训练最终
    scaler = StandardScaler()
    X_full_s = scaler.fit_transform(X.fillna(X.median()))
    final = best_fn()
    final.fit(X_full_s, y)

    # Mar-Apr评估
    if len(X_mar)>0:
        Xm_s = scaler.transform(X_mar.fillna(X.median()))
        probs = final.predict_proba(Xm_s)[:,1]
        mkt   = X_mar['mkt_fair'].values if 'mkt_fair' in X_mar.columns else np.full(len(X_mar), 0.5)

        # 市场偏离预测
        preds_edge = np.full(len(probs), -1, dtype=int)
        preds_edge[(probs - mkt) >= EDGE_THRESH] = 1
        preds_edge[(mkt - probs) >= EDGE_THRESH] = 0
        bet_mask_edge = preds_edge != -1

        # 绝对阈值预测
        preds_abs = np.zeros(len(probs), dtype=int)
        preds_abs[probs>=0.65]=1
        preds_abs[probs<=0.35]=0
        bet_mask_abs = (probs>=0.65)|(probs<=0.35)

        # 选择更好策略
        mar_df = df_odds.iloc[mar_idx].reset_index(drop=True)

        for label, preds, bet_mask in [
            ('市场偏离', preds_edge, bet_mask_edge),
            ('绝对阈值', preds_abs, bet_mask_abs),
        ]:
            if bet_mask.sum() < 3: continue
            acc = accuracy_score(y_mar[bet_mask], preds[bet_mask])
            print(f"    {label}: {acc:.1%}(n={int(bet_mask.sum())})")

        # 详细预测
        print(f"\n  Mar-Apr详细预测:")
        for i, (_, r2) in enumerate(mar_df.iterrows()):
            prob = probs[i]
            mkt_val = mkt[i]
            edge_val = prob - mkt_val
            pred = 1 if prob >= 0.65 else (0 if prob <= 0.35 else -1)
            if pred == -1:
                # 市场偏离判断
                if edge_val >= EDGE_THRESH: pred = 1
                elif edge_val <= -EDGE_THRESH: pred = 0
                else: pred = -1
            correct = (pred != -1 and pred == int(r2['is_over']))
            mark = '✓' if correct else ('⚠' if pred==-1 else '✗')
            bet_flag = pred != -1
            tot = int(r2['FTHG'])+int(r2['FTAG']) if pd.notna(r2['FTHG']) else '?'
            bet_mark = '🔴' if bet_flag else '  '
            actual='大' if r2['is_over']==1 else '小'
            pred_label = '大' if pred==1 else ('小' if pred==0 else '-')
            print(f"    {bet_mark}{mark} {str(r2['Date'])[:10]} {str(r2['HomeTeam'])[:12]:12s} vs {str(r2['AwayTeam'])[:12]:12s} "
                  f"| prob={prob:.2f} mkt={mkt_val:.2f} edge={edge_val:+.2f} {pred_label} | {tot}g {actual}")

    # 保存
    out=OUTPUT_DIR/code; out.mkdir(parents=True,exist_ok=True)
    joblib.dump({'model':final,'scaler':scaler,'features':valid,
                 'edge_threshold':EDGE_THRESH,
                 'league':code,'league_name':LEAGUES[code]['name']}, out/'model.pkl')
    config={'league':code,'league_name':LEAGUES[code]['name'],
            'edge_threshold':EDGE_THRESH,'features':valid,'model_name':best_name,
            'wf_bet_acc':float(best['bet_acc']),'wf_bet_count':int(best['bet_count'])}
    with open(out/'config.json','w',encoding='utf-8') as f:
        json.dump(config,f,ensure_ascii=False,indent=2)
    return config


def main():
    codes = sys.argv[1:] if len(sys.argv)>1 else ['d1','sp1','i1']
    results=[]
    for code in codes:
        if code in LEAGUES:
            r=train_league(code)
            results.append(r)
    print(f"\n{'='*60}")
    for r in results:
        if r:
            s='✓达标' if r.get('wf_bet_acc',0)>=0.75 else '✗未达标'
            print(f"  {r['league_name']:6s}: WF投注={r.get('wf_bet_acc',0):.1%} {s}")


if __name__=='__main__':
    main()
