#!/usr/bin/env python3
"""V2: Extended features + extended model pool for stubborn leagues"""
from __future__ import annotations
import warnings
warnings.filterwarnings('ignore')
import joblib
import numpy as np
import pandas as pd
from pathlib import Path
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier
from sklearn.linear_model import LogisticRegression, RidgeClassifier
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler

FEATURE_COLUMNS = [
    'avg_home_odds', 'avg_away_odds', 'home_prob', 'away_prob', 'prob_diff',
    'handicap', 'asian_home_water', 'asian_away_water', 'water_diff', 'handicap_x_water',
    'home_ppg', 'away_ppg', 'ppg_diff', 'home_form', 'away_form', 'form_diff',
    'home_adv', 'adv_diff', 'prob_x_form', 'form_x_ppg', 'prob_x_handicap', 'water_x_prob',
]


def safe_to_datetime(series):
    s1 = pd.to_datetime(series, format='%Y-%m-%d', errors='coerce')
    if s1.notna().sum() > 0:
        return s1.fillna(pd.to_datetime(series, format='%d/%m/%Y', errors='coerce'))
    return pd.to_datetime(series, format='%d/%m/%Y', errors='coerce')


def create_features_v2(df):
    """Extended feature engineering with more odds columns and rolling stats"""
    features = pd.DataFrame(index=df.index)

    # === Core odds features ===
    if 'AvgH' in df.columns:
        features['avg_home_odds'] = df['AvgH'].fillna(2.0)
        features['avg_away_odds'] = df['AvgA'].fillna(3.0)
    elif 'B365H' in df.columns:
        features['avg_home_odds'] = df['B365H'].fillna(2.0)
        features['avg_away_odds'] = df['B365A'].fillna(3.0)
    else:
        features['avg_home_odds'] = 2.0
        features['avg_away_odds'] = 3.0

    # Draw odds
    if 'AvgD' in df.columns:
        features['avg_draw_odds'] = df['AvgD'].fillna(3.0)
        features['draw_prob'] = 1 / features['avg_draw_odds'].replace(0, np.nan)
    else:
        features['avg_draw_odds'] = 3.0
        features['draw_prob'] = 0.33

    # Over/Under 2.5 odds
    if 'Avg>2.5' in df.columns:
        features['over_odds'] = df['Avg>2.5'].fillna(2.0)
        features['under_odds'] = df['Avg<2.5'].fillna(2.0)
        features['over_prob'] = 1 / features['over_odds'].replace(0, np.nan)
        features['ou_ratio'] = features['over_prob'] / features['under_odds'].replace(0, np.nan)

    features['home_prob'] = 1 / features['avg_home_odds'].replace(0, np.nan)
    features['away_prob'] = 1 / features['avg_away_odds'].replace(0, np.nan)
    features['prob_diff'] = features['home_prob'] - features['away_prob']
    features['prob_ratio'] = features['home_prob'] / features['away_prob'].replace(0, 1)
    features['prob_sum'] = features['home_prob'] + features['away_prob'] + features.get('draw_prob', 0.33)

    # === Asian Handicap features ===
    if 'AHh' in df.columns:
        features['handicap'] = df['AHh'].fillna(0)
        features['asian_home_water'] = df.get('B365AHH', pd.Series(0.9, index=df.index)).fillna(0.9)
        features['asian_away_water'] = df.get('B365AHA', pd.Series(0.9, index=df.index)).fillna(0.9)
        if 'AvgAHH' in df.columns:
            features['avg_asian_home_water'] = df['AvgAHH'].fillna(0.9)
            features['avg_asian_away_water'] = df['AvgAHA'].fillna(0.9)
            features['water_avg_diff'] = features['avg_asian_home_water'] - features['avg_asian_away_water']
    else:
        features['handicap'] = 0.0
        features['asian_home_water'] = 0.9
        features['asian_away_water'] = 0.9

    features['water_diff'] = features['asian_home_water'] - features['asian_away_water']
    features['handicap_x_water'] = features['handicap'] * features['asian_home_water']
    features['water_sum'] = features['asian_home_water'] + features['asian_away_water']

    # === Performance features ===
    if 'HomeTeam' in df.columns and 'AwayTeam' in df.columns and 'FTR' in df.columns:
        teams = list(df['HomeTeam'].dropna().unique())

        def calc_ppg(team, idx):
            past = df.iloc[:idx][(df.iloc[:idx]['HomeTeam'] == team) | (df.iloc[:idx]['AwayTeam'] == team)]
            if len(past) == 0:
                return 1.5
            pts = sum(3 if (r['FTR'] == ('H' if r['HomeTeam'] == team else 'A')) else (1 if r['FTR'] == 'D' else 0) for _, r in past.iterrows())
            return pts / len(past)

        def calc_form(team, idx, w=5):
            past = df.iloc[:idx][(df.iloc[:idx]['HomeTeam'] == team) | (df.iloc[:idx]['AwayTeam'] == team)].tail(w)
            if len(past) == 0:
                return 1.5
            pts = sum(3 if (r['FTR'] == ('H' if r['HomeTeam'] == team else 'A')) else (1 if r['FTR'] == 'D' else 0) for _, r in past.iterrows())
            return pts / len(past)

        def calc_recent_wins(team, idx, w=3):
            past = df.iloc[:idx][(df.iloc[:idx]['HomeTeam'] == team) | (df.iloc[:idx]['AwayTeam'] == team)].tail(w)
            if len(past) == 0:
                return 0.5
            wins = sum(1 if (r['FTR'] == 'H' and r['HomeTeam'] == team) or (r['FTR'] == 'A' and r['AwayTeam'] == team) else 0 for _, r in past.iterrows())
            return wins / len(past)

        def calc_gd(team, idx, w=5):
            past = df.iloc[:idx][(df.iloc[:idx]['HomeTeam'] == team) | (df.iloc[:idx]['AwayTeam'] == team)].tail(w)
            if len(past) == 0:
                return 0
            gds = []
            for _, r in past.iterrows():
                if r['HomeTeam'] == team:
                    gds.append(int(r['FTHG']) - int(r['FTAG']))
                else:
                    gds.append(int(r['FTAG']) - int(r['FTHG']))
            return np.mean(gds)

        def calc_win_rate(team, idx, w=10):
            past = df.iloc[:idx][(df.iloc[:idx]['HomeTeam'] == team) | (df.iloc[:idx]['AwayTeam'] == team)].tail(w)
            if len(past) == 0:
                return 0.45
            wins = sum(1 if (r['FTR'] == 'H' and r['HomeTeam'] == team) or (r['FTR'] == 'A' and r['AwayTeam'] == team) else 0 for _, r in past.iterrows())
            return wins / len(past)

        features['home_ppg'] = [calc_ppg(row['HomeTeam'], i) for i, row in df.iterrows()]
        features['away_ppg'] = [calc_ppg(row['AwayTeam'], i) for i, row in df.iterrows()]
        features['ppg_diff'] = features['home_ppg'] - features['away_ppg']
        features['home_form'] = [calc_form(row['HomeTeam'], i) for i, row in df.iterrows()]
        features['away_form'] = [calc_form(row['AwayTeam'], i) for i, row in df.iterrows()]
        features['form_diff'] = features['home_form'] - features['away_form']
        features['home_recent_wins'] = [calc_recent_wins(row['HomeTeam'], i) for i, row in df.iterrows()]
        features['away_recent_wins'] = [calc_recent_wins(row['AwayTeam'], i) for i, row in df.iterrows()]
        features['recent_wins_diff'] = features['home_recent_wins'] - features['away_recent_wins']
        features['home_gd'] = [calc_gd(row['HomeTeam'], i) for i, row in df.iterrows()]
        features['away_gd'] = [calc_gd(row['AwayTeam'], i) for i, row in df.iterrows()]
        features['gd_diff'] = features['home_gd'] - features['away_gd']
        features['home_win_rate'] = [calc_win_rate(row['HomeTeam'], i) for i, row in df.iterrows()]
        features['away_win_rate'] = [calc_win_rate(row['AwayTeam'], i) for i, row in df.iterrows()]
        features['win_rate_diff'] = features['home_win_rate'] - features['away_win_rate']

        home_adv = {t: (df[df['HomeTeam'] == t]['FTR'] == 'H').mean() for t in teams}
        features['home_adv'] = df['HomeTeam'].map(home_adv).fillna(0.45)
    else:
        features['home_ppg'] = 1.5
        features['away_ppg'] = 1.5
        features['ppg_diff'] = 0.0
        features['home_form'] = 1.5
        features['away_form'] = 1.5
        features['form_diff'] = 0.0
        features['home_adv'] = 0.45
        features['home_recent_wins'] = 0.5
        features['away_recent_wins'] = 0.5
        features['recent_wins_diff'] = 0.0
        features['home_gd'] = 0.0
        features['away_gd'] = 0.0
        features['gd_diff'] = 0.0
        features['home_win_rate'] = 0.45
        features['away_win_rate'] = 0.45
        features['win_rate_diff'] = 0.0

    features['adv_diff'] = features['home_adv'] - 0.45

    # === Interaction features ===
    features['prob_x_form'] = features['home_prob'] * features['home_form']
    features['form_x_ppg'] = features['home_form'] * features['home_ppg']
    features['prob_x_handicap'] = features['home_prob'] * features['handicap']
    features['water_x_prob'] = features['asian_home_water'] * features['home_prob']
    features['form_x_handicap'] = features['home_form'] * features['handicap']
    features['ppg_x_handicap'] = features['home_ppg'] * features['handicap']
    features['gd_x_prob'] = features['home_gd'] * features['home_prob']
    features['winrate_x_prob'] = features['home_win_rate'] * features['home_prob']
    features['handicap_x_form'] = features['handicap'] * features['form_diff']

    all_features = [
        'avg_home_odds', 'avg_away_odds', 'home_prob', 'away_prob', 'prob_diff', 'prob_ratio', 'prob_sum',
        'handicap', 'asian_home_water', 'asian_away_water', 'water_diff', 'handicap_x_water', 'water_sum',
        'home_ppg', 'away_ppg', 'ppg_diff', 'home_form', 'away_form', 'form_diff',
        'home_adv', 'adv_diff', 'prob_x_form', 'form_x_ppg', 'prob_x_handicap', 'water_x_prob',
        'form_x_handicap', 'ppg_x_handicap',
        'prob_ratio', 'water_sum', 'recent_wins_diff', 'form_x_handicap', 'ppg_x_handicap',
        'draw_prob', 'over_prob', 'ou_ratio',
        'avg_asian_home_water', 'avg_asian_away_water', 'water_avg_diff',
        'home_gd', 'away_gd', 'gd_diff',
        'home_win_rate', 'away_win_rate', 'win_rate_diff',
        'gd_x_prob', 'winrate_x_prob', 'handicap_x_form',
    ]

    result = features.fillna(0).replace([np.inf, -np.inf], 0)
    # Only keep columns that exist
    cols = [c for c in all_features if c in result.columns]
    return result[cols]


def optimize_threshold(y_test, probs, preds, fallback_bet_acc):
    best_threshold = 0.55
    best_bet_acc = 0.0
    for threshold in np.arange(0.45, 0.75, 0.01):
        mask = (probs > threshold) | (probs < (1 - threshold))
        if mask.sum() >= 10:
            bet_acc = accuracy_score(y_test[mask], preds[mask])
            if bet_acc > best_bet_acc:
                best_bet_acc = float(bet_acc)
                best_threshold = float(threshold)
    if best_bet_acc < 0.5:
        best_bet_acc = float(fallback_bet_acc)
    return best_threshold, best_bet_acc


MODEL_POOL = [
    ('LR_C0.005', LogisticRegression(max_iter=3000, C=0.005, class_weight='balanced')),
    ('LR_C0.01', LogisticRegression(max_iter=3000, C=0.01, class_weight='balanced')),
    ('LR_C0.02', LogisticRegression(max_iter=3000, C=0.02, class_weight='balanced')),
    ('LR_C0.05', LogisticRegression(max_iter=3000, C=0.05, class_weight='balanced')),
    ('LR_C0.1', LogisticRegression(max_iter=3000, C=0.1, class_weight='balanced')),
    ('LR_C0.2', LogisticRegression(max_iter=3000, C=0.2, class_weight='balanced')),
    ('LR_C0.5', LogisticRegression(max_iter=3000, C=0.5, class_weight='balanced')),
    ('RF_100_3', RandomForestClassifier(n_estimators=100, max_depth=3, random_state=42, n_jobs=-1)),
    ('RF_100_5', RandomForestClassifier(n_estimators=100, max_depth=5, random_state=42, n_jobs=-1)),
    ('RF_200_3', RandomForestClassifier(n_estimators=200, max_depth=3, random_state=42, n_jobs=-1)),
    ('RF_200_5', RandomForestClassifier(n_estimators=200, max_depth=5, random_state=42, n_jobs=-1)),
    ('RF_200_7', RandomForestClassifier(n_estimators=200, max_depth=7, random_state=42, n_jobs=-1)),
    ('RF_300_5', RandomForestClassifier(n_estimators=300, max_depth=5, random_state=42, n_jobs=-1)),
    ('RF_300_7', RandomForestClassifier(n_estimators=300, max_depth=7, random_state=42, n_jobs=-1)),
    ('GB_0.01_3', GradientBoostingClassifier(n_estimators=200, max_depth=3, learning_rate=0.01, random_state=42)),
    ('GB_0.02_3', GradientBoostingClassifier(n_estimators=200, max_depth=3, learning_rate=0.02, random_state=42)),
    ('GB_0.05_3', GradientBoostingClassifier(n_estimators=200, max_depth=3, learning_rate=0.05, random_state=42)),
    ('GB_0.05_4', GradientBoostingClassifier(n_estimators=200, max_depth=4, learning_rate=0.05, random_state=42)),
    ('GB_0.1_3', GradientBoostingClassifier(n_estimators=200, max_depth=3, learning_rate=0.1, random_state=42)),
    ('ET_100_5', ExtraTreesClassifier(n_estimators=100, max_depth=5, random_state=42, n_jobs=-1)),
    ('ET_200_5', ExtraTreesClassifier(n_estimators=200, max_depth=5, random_state=42, n_jobs=-1)),
    ('ET_200_7', ExtraTreesClassifier(n_estimators=200, max_depth=7, random_state=42, n_jobs=-1)),
    ('ET_300_5', ExtraTreesClassifier(n_estimators=300, max_depth=5, random_state=42, n_jobs=-1)),
]


def train_tune(code, data_path, output_dir):
    df = pd.read_csv(data_path, low_memory=False)
    df['date_obj'] = safe_to_datetime(df['Date'])
    df = df.dropna(subset=['date_obj']).sort_values('date_obj').reset_index(drop=True)
    df = df.dropna(subset=['FTHG', 'FTAG'])

    features = create_features_v2(df)
    feature_names = features.columns.tolist()

    if 'AHh' in df.columns:
        df['gd_h'] = df['FTHG'] - df['FTAG'] + df['AHh'].fillna(0)
    else:
        df['gd_h'] = df['FTHG'] - df['FTAG']
    df['asian_result'] = (df['gd_h'] > 0).astype(int)
    df.loc[df['gd_h'] == 0, 'asian_result'] = 0

    split_idx = int(len(df) * 0.8)
    X_train = features.iloc[:split_idx]
    X_test = features.iloc[split_idx:]
    y_train = df['asian_result'].iloc[:split_idx]
    y_test = df['asian_result'].iloc[split_idx:]

    scaler = StandardScaler()
    X_train_scaled = scaler.fit_transform(X_train)
    X_test_scaled = scaler.transform(X_test)

    best_model = None
    best_name = ''
    best_bet_acc = 0.0
    best_threshold = 0.55
    best_probs = None
    best_preds = None
    results = []

    for name, model in MODEL_POOL:
        try:
            model.fit(X_train_scaled, y_train)
            preds = model.predict(X_test_scaled)
            probs = model.predict_proba(X_test_scaled)[:, 1]
            test_acc = accuracy_score(y_test, preds)
            threshold, bet_acc = optimize_threshold(y_test, probs, preds, test_acc)
            results.append((name, bet_acc, test_acc, threshold, len(probs)))
            if bet_acc > best_bet_acc:
                best_bet_acc = float(bet_acc)
                best_threshold = float(threshold)
                best_model = model
                best_name = name
                best_probs = probs
                best_preds = preds
        except Exception as e:
            continue

    results.sort(key=lambda x: x[1], reverse=True)
    print(f'\n=== {code} Top 10 Models ===')
    for name, bet_acc, test_acc, threshold, n in results[:10]:
        print(f'  {name}: bet_acc={bet_acc*100:.2f}%, test_acc={test_acc*100:.2f}%, thresh={threshold:.2f}')

    league_names = {'sc1': '苏超', 'cz1': '捷克甲级', 'd1': '德甲'}
    league_name = league_names.get(code, code)

    high_conf_mask = best_probs > 0.70
    high_conf_acc = float(accuracy_score(y_test[high_conf_mask], best_preds[high_conf_mask])) if high_conf_mask.sum() > 0 else float(best_bet_acc)

    output_dir.mkdir(parents=True, exist_ok=True)
    joblib.dump(scaler, output_dir / f'{code}_pro_v4_scaler.pkl')
    joblib.dump(best_model, output_dir / f'{code}_pro_v4_model.pkl')
    joblib.dump({
        'threshold': best_threshold,
        'bet_acc': float(best_bet_acc),
        'high_conf_acc': float(high_conf_acc),
        'high_conf_count': int((best_probs > 0.70).sum()),
        'features': feature_names,
        'model_name': best_name,
        'data_count': int(len(df)),
        'league': league_name,
        'code': code,
    }, output_dir / f'{code}_pro_v4_config.pkl')

    print(f'>>> {code} BEST: {best_name} - bet_acc={best_bet_acc*100:.2f}%')
    return best_bet_acc


if __name__ == '__main__':
    output_dir = Path('models')
    codes = [('sc1', 'historical_data/sc1_history.csv'),
             ('cz1', 'historical_data/cz1_history.csv'),
             ('d1', 'historical_data/d1_history.csv')]

    for code, path in codes:
        print(f'\n--- Tuning V2: {code} ---')
        acc = train_tune(code, path, output_dir)
        print(f'Result: {acc*100:.2f}%')
