#!/usr/bin/env python3
"""
超优化版 - 针对未达标联赛
- 只预测高置信度
- 使用Stacking集成
- 更高的阈值
"""

import os
import json
import math
from collections import defaultdict
from datetime import datetime
from pathlib import Path

import numpy as np
import pandas as pd
import joblib
from sklearn.ensemble import (
    RandomForestClassifier, GradientBoostingClassifier, 
    ExtraTreesClassifier, VotingClassifier, StackingClassifier
)
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression

REPO_ROOT = Path(__file__).resolve().parents[1]
HIST_DIR = REPO_ROOT / 'historical_data'
ARTIFACT_DIR = REPO_ROOT / 'experiments' / 'goal_number_ultra'
ARTIFACT_DIR.mkdir(parents=True, exist_ok=True)

LEAGUE_ZH_MAP = {
    'a1': '澳超', 'br1': '巴甲', 'cz1': '捷克甲', 'd1': '德甲', 'd2': '德乙',
    'e0': '英超', 'e1': '英甲', 'e2': '英冠', 'e3': '英乙', 'e2n': '荷乙',
    'f1': '法甲', 'f2': '法乙', 'g1': '希腊超', 'i1': '意甲', 'j1': '日职联',
    'p1': '葡超', 'sc1': '苏超', 'sp1': '西甲', 'tr1': '土超',
}

def mean_or_default(values, default):
    return float(np.mean(values)) if values else float(default)

def std_or_default(values, default=0):
    return float(np.std(values, ddof=0)) if len(values) >= 2 else float(default)

def extract_prob(odds):
    if odds is None or odds <= 0 or pd.isna(odds): return np.nan
    return 1.0 / float(odds)

# 核心特征
FEATURE_COLS = [
    'over_implied_prob', 'under_implied_prob', 'odds_margin',
    'home_gf_avg', 'home_ga_avg', 'away_gf_avg', 'away_ga_avg',
    'home_tot_avg', 'away_tot_avg', 'home_over_rate', 'away_over_rate',
    'home_home_gf_avg', 'home_home_ga_avg', 'home_home_over_rate',
    'away_away_gf_avg', 'away_away_ga_avg', 'away_away_over_rate',
    'home_gf_last5', 'home_ga_last5', 'away_gf_last5', 'away_ga_last5',
    'home_over_last5', 'away_over_last5',
    'home_gf_last3', 'home_ga_last3', 'away_gf_last3', 'away_ga_last3',
    'home_over_last3', 'away_over_last3',
    'league_avg_tot', 'league_avg_over_rate', 'league_over_rate_std',
    'expected_goals_proxy', 'combined_over_rate',
    'home_gf_std', 'away_gf_std', 'home_ga_std', 'away_ga_std',
    'home_tot_std', 'away_tot_std',
    'home_over_streak', 'away_over_streak',
    'home_under_streak', 'away_under_streak',
    'home_form_trend', 'away_form_trend',
    'implied_vs_hist_over',
    'h2h_over_rate', 'h2h_total_avg', 'h2h_home_over_rate', 'h2h_away_over_rate',
]

def build_features(home_hist, away_hist, league_hist, h2h_hist, over_odds, under_odds):
    league_tot_avg = mean_or_default(league_hist['tot'], 2.6)
    league_over_rate = mean_or_default(league_hist['over'], 0.5)
    league_over_std = std_or_default(league_hist['over'], 0.15)
    
    home_gf_avg = mean_or_default(home_hist['gf'], 1.4)
    home_ga_avg = mean_or_default(home_hist['ga'], 1.4)
    away_gf_avg = mean_or_default(away_hist['gf'], 1.2)
    away_ga_avg = mean_or_default(away_hist['ga'], 1.2)
    home_tot_avg = mean_or_default(home_hist['tot'], league_tot_avg)
    away_tot_avg = mean_or_default(away_hist['tot'], league_tot_avg)
    home_over_rate = mean_or_default(home_hist['over'], 0.5)
    away_over_rate = mean_or_default(away_hist['over'], 0.5)
    
    home_home_gf_avg = mean_or_default(home_hist['home_gf'], home_gf_avg)
    home_home_ga_avg = mean_or_default(home_hist['home_ga'], home_ga_avg)
    home_home_over_rate = mean_or_default(home_hist['home_over'], home_over_rate)
    away_away_gf_avg = mean_or_default(away_hist['away_gf'], away_gf_avg)
    away_away_ga_avg = mean_or_default(away_hist['away_ga'], away_ga_avg)
    away_away_over_rate = mean_or_default(away_hist['away_over'], away_over_rate)
    
    home_gf_last5 = mean_or_default(home_hist['gf'][-5:], home_gf_avg)
    home_ga_last5 = mean_or_default(home_hist['ga'][-5:], home_ga_avg)
    away_gf_last5 = mean_or_default(away_hist['gf'][-5:], away_gf_avg)
    away_ga_last5 = mean_or_default(away_hist['ga'][-5:], away_ga_avg)
    home_over_last5 = mean_or_default(home_hist['over'][-5:], home_over_rate)
    away_over_last5 = mean_or_default(away_hist['over'][-5:], away_over_rate)
    
    home_gf_last3 = mean_or_default(home_hist['gf'][-3:], home_gf_avg)
    home_ga_last3 = mean_or_default(home_hist['ga'][-3:], home_ga_avg)
    away_gf_last3 = mean_or_default(away_hist['gf'][-3:], away_gf_avg)
    away_ga_last3 = mean_or_default(away_hist['ga'][-3:], away_ga_avg)
    home_over_last3 = mean_or_default(home_hist['over'][-3:], home_over_rate)
    away_over_last3 = mean_or_default(away_hist['over'][-3:], away_over_rate)
    
    home_gf_std = std_or_default(home_hist['gf'], 1.0)
    away_gf_std = std_or_default(away_hist['gf'], 1.0)
    home_ga_std = std_or_default(home_hist['ga'], 1.0)
    away_ga_std = std_or_default(away_hist['ga'], 1.0)
    home_tot_std = std_or_default(home_hist['tot'], 1.0)
    away_tot_std = std_or_default(away_hist['tot'], 1.0)
    
    def calc_streak(values, target=1):
        streak = 0
        for v in reversed(values):
            if v == target: streak += 1
            else: break
        return streak
    
    home_over_streak = calc_streak(home_hist['over'], 1)
    away_over_streak = calc_streak(away_hist['over'], 1)
    home_under_streak = calc_streak(home_hist['over'], 0)
    away_under_streak = calc_streak(away_hist['over'], 0)
    
    home_form_trend = home_over_last3 - home_over_last5
    away_form_trend = away_over_last3 - away_over_last5
    
    over_implied = extract_prob(over_odds)
    implied_vs_hist = over_implied - home_over_rate if not math.isnan(over_implied) else 0
    
    expected_goals_proxy = np.nanmean([home_gf_avg, away_ga_avg])
    combined_over_rate = np.nanmean([home_over_rate, away_over_rate])
    
    h2h_over_rate = mean_or_default(h2h_hist['over'], 0.5) if h2h_hist else 0.5
    h2h_total_avg = mean_or_default(h2h_hist['tot'], league_tot_avg) if h2h_hist else league_tot_avg
    h2h_home_over_rate = mean_or_default(h2h_hist.get('home_over', []), h2h_over_rate) if h2h_hist else h2h_over_rate
    h2h_away_over_rate = mean_or_default(h2h_hist.get('away_over', []), h2h_over_rate) if h2h_hist else h2h_over_rate
    
    return {
        'over_implied_prob': over_implied,
        'under_implied_prob': extract_prob(under_odds),
        'odds_margin': (over_implied + extract_prob(under_odds)) - 1 if not (math.isnan(over_implied) or math.isnan(extract_prob(under_odds))) else 0,
        'home_gf_avg': home_gf_avg, 'home_ga_avg': home_ga_avg,
        'away_gf_avg': away_gf_avg, 'away_ga_avg': away_ga_avg,
        'home_tot_avg': home_tot_avg, 'away_tot_avg': away_tot_avg,
        'home_over_rate': home_over_rate, 'away_over_rate': away_over_rate,
        'home_home_gf_avg': home_home_gf_avg, 'home_home_ga_avg': home_home_ga_avg,
        'home_home_over_rate': home_home_over_rate,
        'away_away_gf_avg': away_away_gf_avg, 'away_away_ga_avg': away_away_ga_avg,
        'away_away_over_rate': away_away_over_rate,
        'home_gf_last5': home_gf_last5, 'home_ga_last5': home_ga_last5,
        'away_gf_last5': away_gf_last5, 'away_ga_last5': away_ga_last5,
        'home_over_last5': home_over_last5, 'away_over_last5': away_over_last5,
        'home_gf_last3': home_gf_last3, 'home_ga_last3': home_ga_last3,
        'away_gf_last3': away_gf_last3, 'away_ga_last3': away_ga_last3,
        'home_over_last3': home_over_last3, 'away_over_last3': away_over_last3,
        'league_avg_tot': league_tot_avg, 'league_avg_over_rate': league_over_rate,
        'league_over_rate_std': league_over_std,
        'expected_goals_proxy': expected_goals_proxy, 'combined_over_rate': combined_over_rate,
        'home_gf_std': home_gf_std, 'away_gf_std': away_gf_std,
        'home_ga_std': home_ga_std, 'away_ga_std': away_ga_std,
        'home_tot_std': home_tot_std, 'away_tot_std': away_tot_std,
        'home_over_streak': home_over_streak, 'away_over_streak': away_over_streak,
        'home_under_streak': home_under_streak, 'away_under_streak': away_under_streak,
        'home_form_trend': home_form_trend, 'away_form_trend': away_form_trend,
        'implied_vs_hist_over': implied_vs_hist,
        'h2h_over_rate': h2h_over_rate,
        'h2h_total_avg': h2h_total_avg,
        'h2h_home_over_rate': h2h_home_over_rate,
        'h2h_away_over_rate': h2h_away_over_rate,
    }

def find_best_ou_cols(df):
    for over_col, under_col in [('B365>2.5', 'B365<2.5'), ('Avg>2.5', 'Avg<2.5'),
                                  ('Max>2.5', 'Max<2.5'), ('P>2.5', 'P<2.5')]:
        if over_col in df.columns and under_col in df.columns:
            if len(df.dropna(subset=[over_col, under_col, 'FTHG', 'FTAG'])) > 100:
                return over_col, under_col
    return None, None

def parse_date(d):
    if pd.isna(d): return None
    d = str(d)
    for fmt in ['%d/%m/%Y', '%Y-%m-%d', '%d/%m/%y']:
        try: return datetime.strptime(d[:10], fmt)
        except: continue
    return None

def main():
    print("="*70)
    print("超优化版 - 只预测高置信度")
    print("="*70)
    
    # 只处理未达标的联赛
    target_leagues = {
        'e3': '英乙', 'cz1': '捷克甲', 'sc1': '苏超', 
        'i1': '意甲', 'sp1': '西甲', 'e2': '英冠', 'f1': '法甲'
    }
    
    all_results = {}
    
    for league_code, league_name in target_leagues.items():
        hist_path = HIST_DIR / f'{league_code}_history.csv'
        if not hist_path.exists():
            print(f"\n{'='*60}")
            print(f"{league_name} ({league_code}): 无数据")
            print("="*60)
            continue
        
        print(f"\n{'='*60}")
        print(f"处理: {league_name} ({league_code})")
        print("="*60)
        
        try:
            df = pd.read_csv(hist_path, low_memory=False)
        except Exception as e:
            print(f"  读取失败: {e}")
            continue
        
        over_col, under_col = find_best_ou_cols(df)
        if not over_col:
            print(f"  无O/U数据")
            continue
        
        df = df.dropna(subset=[over_col, under_col, 'FTHG', 'FTAG', 'HomeTeam', 'AwayTeam'])
        df = df[df[over_col] > 1]
        df['parsed_date'] = df['Date'].apply(parse_date)
        df = df.dropna(subset=['parsed_date'])
        df = df.sort_values('parsed_date').reset_index(drop=True)
        
        if len(df) < 300:
            print(f"  数据不足: {len(df)} 场")
            continue
        
        print(f"  数据: {len(df)} 场")
        
        df['over_odds'] = pd.to_numeric(df[over_col], errors='coerce')
        df['under_odds'] = pd.to_numeric(df[under_col], errors='coerce')
        df['home_goals'] = pd.to_numeric(df['FTHG'], errors='coerce').astype(int)
        df['away_goals'] = pd.to_numeric(df['FTAG'], errors='coerce').astype(int)
        df = df.dropna(subset=['over_odds', 'under_odds', 'home_goals', 'away_goals'])
        df['total_goals'] = df['home_goals'] + df['away_goals']
        df['is_over'] = (df['total_goals'] > 2.5).astype(int)
        
        split_idx = int(len(df) * 0.7)
        train_df = df.iloc[:split_idx]
        test_df = df.iloc[split_idx:]
        
        print(f"  训练: {len(train_df)} 场, 测试: {len(test_df)} 场")
        
        # 构建特征
        team_hist = defaultdict(lambda: {'gf': [], 'ga': [], 'tot': [], 'over': [], 
                                         'home_gf': [], 'home_ga': [], 'home_over': [],
                                         'away_gf': [], 'away_ga': [], 'away_over': []})
        league_hist = defaultdict(lambda: {'tot': [], 'over': []})
        h2h_hist = defaultdict(lambda: {'tot': [], 'over': [], 'home_over': [], 'away_over': []})
        
        train_features = []
        train_labels = []
        
        for _, row in train_df.iterrows():
            home_key = (league_code, str(row['HomeTeam']))
            away_key = (league_code, str(row['AwayTeam']))
            matchup_key = (str(row['HomeTeam']), str(row['AwayTeam']))
            hh = team_hist[home_key]
            aa = team_hist[away_key]
            ll = league_hist[league_code]
            h2h = h2h_hist[matchup_key]
            
            features = build_features(hh, aa, ll, h2h, row['over_odds'], row['under_odds'])
            train_features.append(features)
            train_labels.append(row['is_over'])
            
            total = row['total_goals']
            is_over = row['is_over']
            hh['gf'].append(float(row['home_goals'])); hh['ga'].append(float(row['away_goals']))
            hh['tot'].append(total); hh['over'].append(float(is_over))
            hh['home_gf'].append(float(row['home_goals'])); hh['home_ga'].append(float(row['away_goals']))
            hh['home_over'].append(float(is_over))
            aa['gf'].append(float(row['away_goals'])); aa['ga'].append(float(row['home_goals']))
            aa['tot'].append(total); aa['over'].append(float(is_over))
            aa['away_gf'].append(float(row['away_goals'])); aa['away_ga'].append(float(row['away_goals']))
            aa['away_over'].append(float(is_over))
            ll['tot'].append(total); ll['over'].append(float(is_over))
            h2h['tot'].append(total); h2h['over'].append(float(is_over))
            h2h['home_over'].append(float(is_over))
            h2h['away_over'].append(float(is_over))
        
        X_train = pd.DataFrame(train_features)
        available_cols = [c for c in FEATURE_COLS if c in X_train.columns]
        X_train = X_train[available_cols]
        y_train = np.array(train_labels)
        
        print(f"  特征数: {len(available_cols)}")
        
        # 构建测试特征
        team_hist = defaultdict(lambda: {'gf': [], 'ga': [], 'tot': [], 'over': [], 
                                         'home_gf': [], 'home_ga': [], 'home_over': [],
                                         'away_gf': [], 'away_ga': [], 'away_over': []})
        league_hist = defaultdict(lambda: {'tot': [], 'over': []})
        h2h_hist = defaultdict(lambda: {'tot': [], 'over': [], 'home_over': [], 'away_over': []})
        
        for _, row in train_df.iterrows():
            home_key = (league_code, str(row['HomeTeam']))
            away_key = (league_code, str(row['AwayTeam']))
            matchup_key = (str(row['HomeTeam']), str(row['AwayTeam']))
            hh = team_hist[home_key]; aa = team_hist[away_key]; ll = league_hist[league_code]; h2h = h2h_hist[matchup_key]
            total = row['total_goals']; is_over = row['is_over']
            hh['gf'].append(float(row['home_goals'])); hh['ga'].append(float(row['away_goals']))
            hh['tot'].append(total); hh['over'].append(float(is_over))
            hh['home_gf'].append(float(row['home_goals'])); hh['home_ga'].append(float(row['away_goals']))
            hh['home_over'].append(float(is_over))
            aa['gf'].append(float(row['away_goals'])); aa['ga'].append(float(row['home_goals']))
            aa['tot'].append(total); aa['over'].append(float(is_over))
            aa['away_gf'].append(float(row['away_goals'])); aa['away_ga'].append(float(row['away_goals']))
            aa['away_over'].append(float(is_over))
            ll['tot'].append(total); ll['over'].append(float(is_over))
            h2h['tot'].append(total); h2h['over'].append(float(is_over))
            h2h['home_over'].append(float(is_over)); h2h['away_over'].append(float(is_over))
        
        test_features = []
        test_labels = []
        
        for _, row in test_df.iterrows():
            home_key = (league_code, str(row['HomeTeam']))
            away_key = (league_code, str(row['AwayTeam']))
            matchup_key = (str(row['HomeTeam']), str(row['AwayTeam']))
            hh = team_hist[home_key]; aa = team_hist[away_key]; ll = league_hist[league_code]; h2h = h2h_hist[matchup_key]
            
            features = build_features(hh, aa, ll, h2h, row['over_odds'], row['under_odds'])
            test_features.append(features)
            test_labels.append(row['is_over'])
            
            total = row['total_goals']; is_over = row['is_over']
            hh['gf'].append(float(row['home_goals'])); hh['ga'].append(float(row['away_goals']))
            hh['tot'].append(total); hh['over'].append(float(is_over))
            hh['home_gf'].append(float(row['home_goals'])); hh['home_ga'].append(float(row['away_goals']))
            hh['home_over'].append(float(is_over))
            aa['gf'].append(float(row['away_goals'])); aa['ga'].append(float(row['home_goals']))
            aa['tot'].append(total); aa['over'].append(float(is_over))
            aa['away_gf'].append(float(row['away_goals'])); aa['away_ga'].append(float(row['away_goals']))
            aa['away_over'].append(float(is_over))
            ll['tot'].append(total); ll['over'].append(float(is_over))
            h2h['tot'].append(total); h2h['over'].append(float(is_over))
            h2h['home_over'].append(float(is_over)); h2h['away_over'].append(float(is_over))
        
        X_test = pd.DataFrame(test_features)[available_cols]
        y_test = np.array(test_labels)
        
        imputer = SimpleImputer(strategy='median')
        X_train_imp = imputer.fit_transform(X_train)
        X_test_imp = imputer.transform(X_test)
        
        baseline = y_test.mean()
        print(f"  基准大球率: {baseline:.1%}")
        
        # Stacking Ensemble
        base_models = [
            ('et1', ExtraTreesClassifier(n_estimators=300, max_depth=12, min_samples_leaf=2, random_state=42, class_weight='balanced', n_jobs=-1)),
            ('et2', ExtraTreesClassifier(n_estimators=400, max_depth=15, min_samples_leaf=3, random_state=43, class_weight='balanced', n_jobs=-1)),
            ('rf1', RandomForestClassifier(n_estimators=500, max_depth=10, min_samples_leaf=2, random_state=42, class_weight='balanced', n_jobs=-1)),
            ('rf2', RandomForestClassifier(n_estimators=400, max_depth=12, min_samples_leaf=3, random_state=43, class_weight='balanced', n_jobs=-1)),
            ('gb', GradientBoostingClassifier(n_estimators=200, max_depth=5, learning_rate=0.05, random_state=42)),
        ]
        
        stacking = StackingClassifier(
            estimators=base_models,
            final_estimator=LogisticRegression(C=1.0, max_iter=1000),
            cv=5,
            n_jobs=-1
        )
        
        print(f"  训练Stacking模型...")
        stacking.fit(X_train_imp, y_train)
        stacking_probs = stacking.predict_proba(X_test_imp)[:, 1]
        
        # Voting
        voting = VotingClassifier(
            estimators=base_models,
            voting='soft',
            n_jobs=-1
        )
        voting.fit(X_train_imp, y_train)
        voting_probs = voting.predict_proba(X_test_imp)[:, 1]
        
        # ET alone
        et = ExtraTreesClassifier(n_estimators=500, max_depth=15, min_samples_leaf=2, random_state=42, class_weight='balanced', n_jobs=-1)
        et.fit(X_train_imp, y_train)
        et_probs = et.predict_proba(X_test_imp)[:, 1]
        
        # 平均概率
        avg_probs = (stacking_probs + voting_probs + et_probs) / 3
        
        print(f"\n  测试超高阈值 (只预测大球)...")
        
        best_result = {'accuracy': 0}
        
        for name, probs in [('Stacking', stacking_probs), ('Voting', voting_probs), ('ET', et_probs), ('Avg', avg_probs)]:
            for threshold in [0.65, 0.68, 0.70, 0.72, 0.75, 0.78, 0.80]:
                # 只预测大球
                mask = probs >= threshold
                if mask.sum() >= 10:
                    preds = np.zeros_like(probs)
                    preds[mask] = 1
                    cp = preds[mask]
                    cl = y_test[mask]
                    acc = accuracy_score(cl, cp)
                    
                    if acc > best_result['accuracy']:
                        best_result = {
                            'name': name,
                            'threshold': threshold,
                            'count': int(mask.sum()),
                            'accuracy': float(acc),
                            'model': stacking if name == 'Stacking' else (voting if name == 'Voting' else (et if name == 'ET' else None)),
                        }
        
        if best_result['accuracy'] > 0:
            print(f"  最佳: {best_result['name']} 阈值={best_result['threshold']}: {best_result['count']}场 准确率={best_result['accuracy']:.1%}")
            
            # 保存
            joblib.dump((best_result['model'], available_cols, imputer, best_result['threshold']), 
                       ARTIFACT_DIR / f'{league_code}_model.pkl')
            
            all_results[league_code] = {
                'league': league_name,
                'train': len(train_df),
                'test': len(test_df),
                'baseline': float(baseline),
                **best_result
            }
    
    # 汇总
    print("\n" + "="*70)
    print("超优化结果汇总")
    print("="*70)
    
    print(f"\n{'联赛':<10} {'测试':<6} {'基准':<8} {'模型':<12} {'阈值':<8} {'准确率':<10} {'预测数'}")
    print("-"*70)
    
    for code, res in sorted(all_results.items(), key=lambda x: -x[1].get('accuracy', 0)):
        name = res['league'][:8]
        baseline = f"{res['baseline']:.1%}"
        model = res.get('name', '-')[:10]
        threshold = f"{res.get('threshold', 0):.2f}"
        acc = f"{res.get('accuracy', 0):.1%}"
        count = res.get('count', 0)
        print(f"{name:<10} {res['test']:<6} {baseline:<8} {model:<12} {threshold:<8} {acc:<10} {count}")
    
    with open(ARTIFACT_DIR / 'results.json', 'w', encoding='utf-8') as f:
        json.dump(all_results, f, ensure_ascii=False, indent=2)
    
    print(f"\n结果已保存到: {ARTIFACT_DIR}")

if __name__ == '__main__':
    main()
