#!/usr/bin/env python3
"""
使用历史数据(2023-2025)训练分联赛大小球模型
历史数据只有固定2.5盘口，但数据量充足
"""

import os
import json
import math
import re
from collections import defaultdict
from datetime import datetime
from pathlib import Path

import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline

REPO_ROOT = Path(__file__).resolve().parents[1]
HIST_DIR = REPO_ROOT / 'historical_data'
ARTIFACT_DIR = REPO_ROOT / 'experiments' / 'goal_number_historical_per_league'
ARTIFACT_DIR.mkdir(parents=True, exist_ok=True)

LEAGUE_ZH_MAP = {
    'a1': '澳超', 'br1': '巴甲', 'cz1': '捷克甲', 'd1': '德甲', 'd2': '德乙',
    'e0': '英超', 'e1': '英甲', 'e2': '英冠', 'e3': '英乙', 'e2n': '荷乙',
    'f1': '法甲', 'f2': '法乙', 'g1': '希腊超', 'i1': '意甲', 'j1': '日职联',
    'j2': '日职乙', 'k2': '韩国K2', 'm1': '美职联', 'p1': '葡超', 'r1': '俄超',
    'sc1': '苏超', 'sp1': '西甲', 'tr1': '土超', 'uz1': '乌兹别克超',
}

def mean_or_default(values, default):
    return float(np.mean(values)) if values else float(default)

def extract_prob(odds):
    if odds is None or odds <= 0 or pd.isna(odds):
        return np.nan
    return 1.0 / float(odds)

# 特征列表
FEATURE_COLS = [
    # O/U odds
    'over_odds', 'under_odds', 'over_implied_prob', 'under_implied_prob', 'odds_margin',
    # Historical team stats
    'home_gf_avg', 'home_ga_avg', 'away_gf_avg', 'away_ga_avg',
    'home_tot_avg', 'away_tot_avg', 'home_over_rate', 'away_over_rate',
    'home_home_gf_avg', 'home_home_ga_avg', 'away_away_gf_avg', 'away_away_ga_avg',
    'home_home_over_rate', 'away_away_over_rate',
    'home_gf_last5', 'home_ga_last5', 'away_gf_last5', 'away_ga_last5',
    'home_over_last5', 'away_over_last5',
    'expected_goals_proxy', 'combined_over_rate',
    'league_avg_tot', 'league_avg_over_rate',
    # Derived
    'form_diff', 'goals_diff', 'over_rate_diff',
]

def build_features(home_hist, away_hist, league_hist):
    league_tot_avg = mean_or_default(league_hist['tot'], 2.6)
    league_over_rate = mean_or_default(league_hist['over'], 0.5)
    
    home_gf_avg = mean_or_default(home_hist['gf'], 1.4)
    home_ga_avg = mean_or_default(home_hist['ga'], 1.4)
    away_gf_avg = mean_or_default(away_hist['gf'], 1.2)
    away_ga_avg = mean_or_default(away_hist['ga'], 1.2)
    home_tot_avg = mean_or_default(home_hist['tot'], league_tot_avg)
    away_tot_avg = mean_or_default(away_hist['tot'], league_tot_avg)
    home_over_rate = mean_or_default(home_hist['over'], 0.5)
    away_over_rate = mean_or_default(away_hist['over'], 0.5)
    home_home_gf_avg = mean_or_default(home_hist['home_gf'], home_gf_avg)
    home_home_ga_avg = mean_or_default(home_hist['home_ga'], home_ga_avg)
    away_away_gf_avg = mean_or_default(away_hist['away_gf'], away_gf_avg)
    away_away_ga_avg = mean_or_default(away_hist['away_ga'], away_ga_avg)
    home_home_over_rate = mean_or_default(home_hist['home_over'], home_over_rate)
    away_away_over_rate = mean_or_default(away_hist['away_over'], away_over_rate)
    home_gf_last5 = mean_or_default(home_hist['gf'][-5:], home_gf_avg)
    home_ga_last5 = mean_or_default(home_hist['ga'][-5:], home_ga_avg)
    away_gf_last5 = mean_or_default(away_hist['gf'][-5:], away_gf_avg)
    away_ga_last5 = mean_or_default(away_hist['ga'][-5:], away_ga_avg)
    home_over_last5 = mean_or_default(home_hist['over'][-5:], home_over_rate)
    away_over_last5 = mean_or_default(away_hist['over'][-5:], away_over_rate)
    expected_goals_proxy = np.nanmean([home_gf_avg, away_ga_avg])
    combined_over_rate = np.nanmean([home_over_rate, away_over_rate])
    
    return {
        'over_odds': 2.0, 'under_odds': 2.0,  # 历史数据没有赔率，用默认值
        'over_implied_prob': 0.5, 'under_implied_prob': 0.5, 'odds_margin': 0,
        'home_gf_avg': home_gf_avg, 'home_ga_avg': home_ga_avg,
        'away_gf_avg': away_gf_avg, 'away_ga_avg': away_ga_avg,
        'home_tot_avg': home_tot_avg, 'away_tot_avg': away_tot_avg,
        'home_over_rate': home_over_rate, 'away_over_rate': away_over_rate,
        'home_home_gf_avg': home_home_gf_avg, 'home_home_ga_avg': home_home_ga_avg,
        'away_away_gf_avg': away_away_gf_avg, 'away_away_ga_avg': away_away_ga_avg,
        'home_home_over_rate': home_home_over_rate, 'away_away_over_rate': away_away_over_rate,
        'home_gf_last5': home_gf_last5, 'home_ga_last5': home_ga_last5,
        'away_gf_last5': away_gf_last5, 'away_ga_last5': away_ga_last5,
        'home_over_last5': home_over_last5, 'away_over_last5': away_over_last5,
        'expected_goals_proxy': expected_goals_proxy, 'combined_over_rate': combined_over_rate,
        'league_avg_tot': league_tot_avg, 'league_avg_over_rate': league_over_rate,
        'form_diff': home_over_rate - away_over_rate,
        'goals_diff': (home_gf_avg + away_ga_avg) - (away_gf_avg + home_ga_avg),
        'over_rate_diff': home_over_rate - away_over_rate,
    }

def find_best_ou_cols(df):
    """找到最好的O/U列"""
    candidates = [
        ('B365>2.5', 'B365<2.5'),
        ('Avg>2.5', 'Avg<2.5'),
        ('Max>2.5', 'Max<2.5'),
        ('P>2.5', 'P<2.5'),
    ]
    for over_col, under_col in candidates:
        if over_col in df.columns and under_col in df.columns:
            valid = df.dropna(subset=[over_col, under_col, 'FTHG', 'FTAG'])
            if len(valid) > 100:
                return over_col, under_col
    return None, None

def main():
    print("="*70)
    print("使用历史数据(2023-2025)训练分联赛大小球模型")
    print("="*70)
    
    # 获取所有历史文件
    hist_files = sorted(HIST_DIR.glob('*_history.csv'))
    print(f"\n找到 {len(hist_files)} 个历史数据文件")
    
    all_results = {}
    
    for hist_path in hist_files:
        league_code = hist_path.stem.replace('_history', '')
        league_name = LEAGUE_ZH_MAP.get(league_code, league_code)
        
        print(f"\n{'='*60}")
        print(f"处理: {league_name} ({league_code})")
        print("="*60)
        
        # 读取数据
        try:
            df = pd.read_csv(hist_path, low_memory=False)
        except Exception as e:
            print(f"  读取失败: {e}")
            continue
        
        # 找到O/U列
        over_col, under_col = find_best_ou_cols(df)
        if not over_col:
            print(f"  无O/U数据，跳过")
            continue
        
        print(f"  使用列: {over_col}")
        
        # 清理数据
        df = df.dropna(subset=[over_col, under_col, 'FTHG', 'FTAG', 'HomeTeam', 'AwayTeam'])
        df = df[df[over_col] > 1]  # 赔率要大于1
        
        if len(df) < 200:
            print(f"  数据不足: {len(df)} 场")
            continue
        
        print(f"  有效数据: {len(df)} 场")
        
        # 处理日期
        def parse_date(d):
            if pd.isna(d): return None
            d = str(d)
            # 尝试不同格式
            for fmt in ['%d/%m/%Y', '%Y-%m-%d', '%d/%m/%y']:
                try:
                    return datetime.strptime(d[:10], fmt)
                except:
                    continue
            return None
        
        df['parsed_date'] = df['Date'].apply(parse_date)
        df = df.dropna(subset=['parsed_date'])
        df = df.sort_values('parsed_date')
        
        if len(df) < 200:
            print(f"  日期解析后数据不足: {len(df)} 场")
            continue
        
        print(f"  日期范围: {df['parsed_date'].iloc[0].strftime('%Y-%m-%d')} ~ {df['parsed_date'].iloc[-1].strftime('%Y-%m-%d')}")
        
        # 转换数值
        df['over_odds'] = pd.to_numeric(df[over_col], errors='coerce')
        df['under_odds'] = pd.to_numeric(df[under_col], errors='coerce')
        df['home_goals'] = pd.to_numeric(df['FTHG'], errors='coerce')
        df['away_goals'] = pd.to_numeric(df['FTAG'], errors='coerce')
        df = df.dropna(subset=['over_odds', 'under_odds', 'home_goals', 'away_goals'])
        df['home_goals'] = df['home_goals'].astype(int)
        df['away_goals'] = df['away_goals'].astype(int)
        df['total_goals'] = df['home_goals'] + df['away_goals']
        df['is_over'] = (df['total_goals'] > 2.5).astype(int)
        
        # 70% 训练, 30% 测试
        split_idx = int(len(df) * 0.7)
        train_df = df.iloc[:split_idx]
        test_df = df.iloc[split_idx:]
        
        print(f"  训练: {len(train_df)} 场, 测试: {len(test_df)} 场")
        
        if len(train_df) < 100 or len(test_df) < 50:
            print(f"  数据不足，跳过")
            continue
        
        # 构建训练特征
        team_hist = defaultdict(lambda: {'gf': [], 'ga': [], 'tot': [], 'over': [], 
                                         'home_gf': [], 'home_ga': [], 'home_over': [],
                                         'away_gf': [], 'away_ga': [], 'away_over': []})
        league_hist = defaultdict(lambda: {'tot': [], 'over': []})
        
        train_features = []
        train_labels = []
        
        for _, row in train_df.iterrows():
            home_key = (league_code, str(row['HomeTeam']))
            away_key = (league_code, str(row['AwayTeam']))
            hh = team_hist[home_key]
            aa = team_hist[away_key]
            ll = league_hist[league_code]
            
            features = build_features(hh, aa, ll)
            features['over_odds'] = row['over_odds']
            features['under_odds'] = row['under_odds']
            features['over_implied_prob'] = extract_prob(row['over_odds'])
            features['under_implied_prob'] = extract_prob(row['under_odds'])
            features['odds_margin'] = (features['over_implied_prob'] + features['under_implied_prob']) - 1
            
            train_features.append(features)
            train_labels.append(row['is_over'])
            
            # 更新历史
            total = row['total_goals']
            is_over = row['is_over']
            hh['gf'].append(float(row['home_goals']))
            hh['ga'].append(float(row['away_goals']))
            hh['tot'].append(total)
            hh['over'].append(float(is_over))
            hh['home_gf'].append(float(row['home_goals']))
            hh['home_ga'].append(float(row['away_goals']))
            hh['home_over'].append(float(is_over))
            aa['gf'].append(float(row['away_goals']))
            aa['ga'].append(float(row['home_goals']))
            aa['tot'].append(total)
            aa['over'].append(float(is_over))
            aa['away_gf'].append(float(row['away_goals']))
            aa['away_ga'].append(float(row['away_goals']))
            aa['away_over'].append(float(is_over))
            ll['tot'].append(total)
            ll['over'].append(float(is_over))
        
        X_train = pd.DataFrame(train_features)[FEATURE_COLS]
        y_train = np.array(train_labels)
        
        print(f"  特征数: {X_train.shape[1]}, 缺失率: {X_train.isnull().sum().sum() / X_train.size:.1%}")
        
        # 训练
        model = Pipeline([
            ('imp', SimpleImputer(strategy='median')),
            ('clf', RandomForestClassifier(
                n_estimators=500, max_depth=12, min_samples_leaf=3,
                random_state=42, class_weight='balanced', n_jobs=-1
            ))
        ])
        model.fit(X_train, y_train)
        
        # 保存模型
        model_path = ARTIFACT_DIR / f'{league_code}_model.pkl'
        joblib.dump((model, FEATURE_COLS), model_path)
        
        # 测试
        team_hist = defaultdict(lambda: {'gf': [], 'ga': [], 'tot': [], 'over': [], 
                                         'home_gf': [], 'home_ga': [], 'home_over': [],
                                         'away_gf': [], 'away_ga': [], 'away_over': []})
        league_hist = defaultdict(lambda: {'tot': [], 'over': []})
        
        # 用训练数据建立历史
        for _, row in train_df.iterrows():
            home_key = (league_code, str(row['HomeTeam']))
            away_key = (league_code, str(row['AwayTeam']))
            hh = team_hist[home_key]
            aa = team_hist[away_key]
            ll = league_hist[league_code]
            total = row['total_goals']
            is_over = row['is_over']
            hh['gf'].append(float(row['home_goals']))
            hh['ga'].append(float(row['away_goals']))
            hh['tot'].append(total)
            hh['over'].append(float(is_over))
            hh['home_gf'].append(float(row['home_goals']))
            hh['home_ga'].append(float(row['away_goals']))
            hh['home_over'].append(float(is_over))
            aa['gf'].append(float(row['away_goals']))
            aa['ga'].append(float(row['home_goals']))
            aa['tot'].append(total)
            aa['over'].append(float(is_over))
            aa['away_gf'].append(float(row['away_goals']))
            aa['away_ga'].append(float(row['away_goals']))
            aa['away_over'].append(float(is_over))
            ll['tot'].append(total)
            ll['over'].append(float(is_over))
        
        test_features = []
        test_labels = []
        
        for _, row in test_df.iterrows():
            home_key = (league_code, str(row['HomeTeam']))
            away_key = (league_code, str(row['AwayTeam']))
            hh = team_hist[home_key]
            aa = team_hist[away_key]
            ll = league_hist[league_code]
            
            features = build_features(hh, aa, ll)
            features['over_odds'] = row['over_odds']
            features['under_odds'] = row['under_odds']
            features['over_implied_prob'] = extract_prob(row['over_odds'])
            features['under_implied_prob'] = extract_prob(row['under_odds'])
            features['odds_margin'] = (features['over_implied_prob'] + features['under_implied_prob']) - 1
            
            test_features.append(features)
            test_labels.append(row['is_over'])
        
        X_test = pd.DataFrame(test_features)[FEATURE_COLS]
        y_test = np.array(test_labels)
        
        probs = model.predict_proba(X_test)[:, 1]
        
        baseline = y_test.mean()
        print(f"\n  测试结果 ({len(test_df)} 场, 基准大球率: {baseline:.1%}):")
        
        results = {'league': league_name, 'train': len(train_df), 'test': len(test_df), 'baseline': float(baseline)}
        
        for threshold in [0.52, 0.55, 0.58, 0.60, 0.65]:
            mask = (probs >= threshold) | (probs <= (1 - threshold))
            if mask.sum() >= 10:
                cp = probs[mask]; cl = y_test[mask]; cp2 = (cp >= threshold).astype(int)
                acc = accuracy_score(cl, cp2)
                over_m = cp2 == 1; under_m = cp2 == 0
                over_acc = accuracy_score(cl[over_m], cp2[over_m]) if over_m.sum() > 0 else 0
                under_acc = accuracy_score(cl[under_m], cp2[under_m]) if under_m.sum() > 0 else 0
                print(f"    阈值={threshold}: {mask.sum()}场 准确率={acc:.1%} (大球{over_m.sum()}场{over_acc:.1%}, 小球{under_m.sum()}场{under_acc:.1%})")
                results[f'threshold_{threshold}'] = {'count': int(mask.sum()), 'acc': float(acc), 'over_count': int(over_m.sum()), 'over_acc': float(over_acc), 'under_count': int(under_m.sum()), 'under_acc': float(under_acc)}
        
        # 特征重要性
        importances = model.named_steps['clf'].feature_importances_
        indices = np.argsort(importances)[::-1][:5]
        print(f"  特征重要性 Top 5:")
        for i, idx in enumerate(indices):
            print(f"    {i+1}. {FEATURE_COLS[idx]}: {importances[idx]:.4f}")
        
        all_results[league_code] = results
    
    # 汇总
    print("\n" + "="*70)
    print("所有联赛汇总")
    print("="*70)
    
    print(f"\n{'联赛':<12} {'训练':<6} {'测试':<6} {'基准':<8} {'0.55准确率':<12} {'0.60准确率':<12}")
    print("-" * 70)
    
    for code, res in sorted(all_results.items(), key=lambda x: -x[1].get('threshold_0.55', {}).get('acc', 0)):
        name = res['league'][:10]
        baseline = f"{res['baseline']:.1%}"
        acc55 = f"{res.get('threshold_0.55', {}).get('acc', 0):.1%}" if 'threshold_0.55' in res else "-"
        acc60 = f"{res.get('threshold_0.60', {}).get('acc', 0):.1%}" if 'threshold_0.60' in res else "-"
        print(f"{name:<12} {res['train']:<6} {res['test']:<6} {baseline:<8} {acc55:<12} {acc60:<12}")
    
    # 保存结果
    output_path = ARTIFACT_DIR / 'all_results.json'
    with open(output_path, 'w', encoding='utf-8') as f:
        json.dump(all_results, f, ensure_ascii=False, indent=2)
    
    print(f"\n结果已保存到: {output_path}")

if __name__ == '__main__':
    main()
