#!/usr/bin/env python3
"""
高效数据更新+模型重训脚本
从API-Football获取近年数据，重训pro_v4模型，汇报准确率
"""
import os, sys, time, json, warnings, joblib
from pathlib import Path
from datetime import date, datetime, timezone, timedelta
from collections import defaultdict

import numpy as np
import pandas as pd
import requests
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.calibration import CalibratedClassifierCV

warnings.filterwarnings('ignore')

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
HIST_DIR = REPO / 'historical_data'
MODEL_OUT = REPO / 'artifacts' / 'v4_models_serving_current'
ODDS_DIR = REPO / 'artifacts' / 'odds_api_history'
MODEL_OUT.mkdir(parents=True, exist_ok=True)

API_KEY = os.environ.get('API_FOOTBALL_KEY', '57b64e857ebeffdc156f0c41d43c7894')
API_BASE = 'https://v3.football.api-sports.io'
HEADERS = {'x-apisports-key': API_KEY}
TZ = timezone(timedelta(hours=8))

# 需要更新的联赛 + 对应的league_id
LEAGUE_IDS = {
    'd1': 78, 'i1': 135, 'sp1': 140, 'f1': 61, 'tr1': 203,
    'p1': 94, 'e0': 39, 'e1': 40, 'e2n': 42, 'sc1': 180,
    'cz1': 0, 'n1': 88, 'g1': 197,
}

# 赛季年份
SEASONS = [2023, 2024, 2025]

def api_get(endpoint, params, sleep=1.0):
    time.sleep(sleep)
    r = requests.get(API_BASE + endpoint, params=params, headers=HEADERS, timeout=30)
    r.raise_for_status()
    return r.json()

def get_season_odds(fixture_ids, league_code):
    """批量获取亚盘赔率"""
    all_ah = {}
    for fid in fixture_ids:
        try:
            data = api_get('/odds', {'fixture': fid, 'timezone': 'Asia/Shanghai'}, sleep=0.3)
            for item in data.get('response', []):
                for bm in item.get('bookmakers', []):
                    for bet in bm.get('bets', []):
                        if bet.get('name') != 'Asian Handicap':
                            continue
                        for val in bet.get('values', []):
                            raw = val.get('value', '')
                            parts = raw.split()
                            if len(parts) < 2:
                                continue
                            side, hcap_str = parts[0], parts[1]
                            try:
                                hcap = float(hcap_str)
                                odd = float(val.get('odd', 0))
                                if odd <= 1:
                                    continue
                                handicap = -hcap if side in ('Home', 'H') else hcap
                                home_odd = odd if side in ('Home', 'H') else odd
                                away_odd = odd if side in ('Away', 'A') else odd
                                if fid not in all_ah:
                                    all_ah[fid] = {'handicap': handicap, 'home_odd': home_odd, 'away_odd': away_odd}
                                else:
                                    if side in ('Home', 'H'):
                                        all_ah[fid]['home_odd'] = odd
                                    else:
                                        all_ah[fid]['away_odd'] = odd
                            except:
                                pass
        except Exception as e:
            pass
    return all_ah

def download_league_data(code, league_id):
    """下载单个联赛的近年数据"""
    print(f"\n{'='*50}\n下载 {code} (league_id={league_id})\n{'='*50}")
    all_fixtures = {}
    
    for season in SEASONS:
        print(f"  Season {season}...", end=' ', flush=True)
        try:
            data = api_get('/fixtures', {
                'league': league_id, 'season': season,
                'timezone': 'Asia/Shanghai'
            }, sleep=1.0)
            fixtures = data.get('response', [])
            print(f"{len(fixtures)} fixtures")
            for item in fixtures:
                fi = item.get('fixture', {})
                st = fi.get('status', {}).get('short', '')
                if st not in ('FT', 'AET', 'PEN'):
                    continue
                fid = fi.get('id')
                teams = item.get('teams', {})
                goals = item.get('goals', {})
                hg = goals.get('home'); ag = goals.get('away')
                if hg is None or ag is None:
                    continue
                ts = fi.get('timestamp')
                d = datetime.fromtimestamp(ts, tz=TZ).strftime('%Y-%m-%d') if ts else f'{season}-01-01'
                all_fixtures[fid] = {
                    'fixture_id': fid, 'Date': d, 'HomeTeam': teams.get('home', {}).get('name', ''),
                    'AwayTeam': teams.get('away', {}).get('name', ''),
                    'FTHG': int(hg), 'FTAG': int(ag),
                }
        except Exception as e:
            print(f"ERROR: {e}")
    
    if not all_fixtures:
        print(f"  无已完成比赛数据")
        return None
    
    print(f"  共 {len(all_fixtures)} 场已结束比赛，获取赔率...", end=' ', flush=True)
    fixture_ids = sorted(all_fixtures.keys())
    ah_data = get_season_odds(fixture_ids, code)
    print(f"获取到 {len(ah_data)} 组赔率")
    
    rows = []
    for fid, fix in all_fixtures.items():
        if fid not in ah_data:
            continue
        ah = ah_data[fid]
        row = {**fix}
        row['B365AHH'] = ah['home_odd']
        row['B365AHA'] = ah['away_odd']
        row['AHh'] = ah['handicap']
        rows.append(row)
    
    if not rows:
        print(f"  无有效赔率数据")
        return None
    
    df = pd.DataFrame(rows)
    return df

def build_features(df):
    """为pro_v4模型构建特征"""
    feats = pd.DataFrame(index=df.index)
    ahh = df['B365AHH'].values.astype(float)
    aha = df['B365AHA'].values.astype(float)
    hcap = df['AHh'].values.astype(float)
    
    feats['home_prob'] = aha / (ahh + aha)
    feats['away_prob'] = ahh / (ahh + aha)
    feats['prob_diff'] = feats['home_prob'] - feats['away_prob']
    feats['handicap'] = hcap
    feats['asian_home_water'] = 1 / ahh
    feats['asian_away_water'] = 1 / aha
    feats['water_diff'] = feats['asian_home_water'] - feats['asian_away_water']
    feats['handicap_x_water'] = hcap * feats['water_diff']
    
    # 计算实际方向
    hg = df['FTHG'].values.astype(float)
    ag = df['FTAG'].values.astype(float)
    hcap_arr = hcap.copy()
    actual = (hg + hcap_arr) > ag  # True=home wins AH
    return feats.fillna(0).replace([np.inf, -np.inf], 0), actual

def train_and_evaluate(code, df_new, df_existing=None):
    """训练并评估单个联赛模型"""
    # 合并数据
    if df_existing is not None and len(df_existing) > 0:
        df_existing = df_existing[
            df_existing['B365AHH'].notna() & (df_existing['B365AHH'] > 1) &
            df_existing['B365AHA'].notna() & (df_existing['B365AHA'] > 1) &
            df_existing['FTHG'].notna() & df_existing['FTAG'].notna()
        ].copy()
        df_combined = pd.concat([df_existing, df_new], ignore_index=True)
    else:
        df_combined = df_new

    df_combined = df_combined[
        df_combined['B365AHH'].notna() & (df_combined['B365AHH'] > 1) &
        df_combined['B365AHA'].notna() & (df_combined['B365AHA'] > 1) &
        df_combined['FTHG'].notna() & df_combined['FTAG'].notna()
    ].copy()
    df_combined['Date'] = pd.to_datetime(df_combined['Date'], errors='coerce')
    df_combined = df_combined[df_combined['Date'].notna()].sort_values('Date')
    
    if len(df_combined) < 50:
        return None, None, len(df_combined)
    
    # 留出最近25%作为测试集
    split_idx = int(len(df_combined) * 0.75)
    train = df_combined.iloc[:split_idx]
    test = df_combined.iloc[split_idx:]
    
    X_train, y_train = build_features(train)
    X_test, y_test = build_features(test)
    
    if len(X_test) < 10:
        return None, None, len(df_combined)
    
    # 训练多个模型，选最好的
    best_acc = 0
    best_model = None
    best_name = ''
    
    for name, clf in [
        ('LR', LogisticRegression(C=1.0, max_iter=1000)),
        ('LR_C0.1', LogisticRegression(C=0.1, max_iter=1000)),
        ('GB', GradientBoostingClassifier(n_estimators=100, max_depth=3, random_state=42)),
        ('RF', RandomForestClassifier(n_estimators=100, max_depth=5, random_state=42)),
    ]:
        try:
            pipe = Pipeline([('scaler', StandardScaler()), ('clf', clf)])
            pipe.fit(X_train.values, y_train)
            pred = pipe.predict(X_test.values)
            acc = accuracy_score(y_test, pred)
            if acc > best_acc:
                best_acc = acc
                best_model = pipe
                best_name = name
        except:
            pass
    
    # 保存模型
    if best_model is not None:
        scaler = StandardScaler()
        X_train_scaled = scaler.fit_transform(X_train.values)
        joblib.dump(best_model, MODEL_OUT / f'{code}_pro_v4_model.pkl')
        joblib.dump(scaler, MODEL_OUT / f'{code}_pro_v4_scaler.pkl')
        cfg = {'feature_mode': 'pro_v4', 'model_name': best_name, 'accuracy': best_acc, 
               'train_samples': len(X_train), 'test_samples': len(X_test),
               'total_samples': len(df_combined), 'data_updated': str(date.today())}
        joblib.dump(cfg, MODEL_OUT / f'{code}_pro_v4_config.pkl')
    
    return best_acc, best_name, len(df_combined)

def main():
    results = []
    
    for code, league_id in LEAGUE_IDS.items():
        print(f"\n{'='*60}\n处理联赛: {code} (id={league_id})\n{'='*60}")
        
        # 读取现有历史数据
        hist_file = HIST_DIR / f'{code}_history.csv'
        df_existing = None
        if hist_file.exists():
            try:
                df_existing = pd.read_csv(hist_file, low_memory=False)
                print(f"  现有历史数据: {len(df_existing)} 行")
            except Exception as e:
                print(f"  读取历史失败: {e}")
                df_existing = None
        
        # 下载新数据
        df_new = download_league_data(code, league_id)
        
        if df_new is None or len(df_new) == 0:
            print(f"  无法获取新数据，跳过")
            continue
        
        print(f"  新数据: {len(df_new)} 场")
        
        # 训练和评估
        acc, model_name, total = train_and_evaluate(code, df_new, df_existing)
        
        if acc is not None:
            results.append({
                'league': code, 'accuracy': acc, 'model': model_name,
                'total_samples': total, 'new_samples': len(df_new)
            })
            print(f"  ✅ {code}: {model_name} 准确率={acc:.1%}, 总样本={total}")
        else:
            results.append({
                'league': code, 'accuracy': None, 'model': 'FAILED',
                'total_samples': total, 'new_samples': len(df_new)
            })
            print(f"  ❌ {code}: 训练失败, 总样本={total}")
    
    # 打印汇总
    print(f"\n{'='*60}")
    print("训练结果汇总")
    print(f"{'='*60}")
    print(f"{'联赛':<8} {'模型':<12} {'准确率':<10} {'总样本'}")
    print("-" * 50)
    for r in sorted(results, key=lambda x: x['accuracy'] or 0, reverse=True):
        acc_str = f"{r['accuracy']:.1%}" if r['accuracy'] else "FAILED"
        print(f"{r['league']:<8} {r['model']:<12} {acc_str:<10} {r['total_samples']}")
    
    # 保存结果
    out_file = REPO / 'models' / 'retrain_results.json'
    with open(out_file, 'w') as f:
        json.dump(results, f, indent=2, ensure_ascii=False)
    print(f"\n结果已保存到 {out_file}")
    return results

if __name__ == '__main__':
    main()
