#!/usr/bin/env python3
"""Train a cross-league goal-number model for leagues without historical O/U odds.

This script:
1. Collects historical data from multiple leagues with O/U odds
2. Trains a general model that can be applied to leagues without O/U odds data
3. Specifically designed to serve R1 (Russia Premier League) and similar leagues
"""

from __future__ import annotations

import json
import os
import re
import shutil
from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Optional, Tuple

import joblib
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

REPO_ROOT = Path(__file__).resolve().parents[1]
HISTORICAL_DIR = REPO_ROOT / 'historical_data'
OUTPUT_DIR = REPO_ROOT / 'experiments' / 'cross_league_goal_number'
LEAGUES_WITH_OU = ['sp1', 'd1', 'e2', 'e1', 'e3', 'f1', 'i1', 'p1', 'sc1', 'tr1', 'cz1', 'e2n']
TARGET_LEAGUES = ['r1', 'br1', 'm1', 'uz1', 'j1', 'g1', 'a1']

OPEN_OU_CANDIDATES: List[Tuple[str, str]] = [
    ("Avg>2.5", "Avg<2.5"),
    ("P>2.5", "P<2.5"),
    ("B365>2.5", "B365<2.5"),
]
CLOSE_OU_CANDIDATES: List[Tuple[str, str]] = [
    ("AvgC>2.5", "AvgC<2.5"),
    ("PC>2.5", "PC<2.5"),
    ("B365C>2.5", "B365C<2.5"),
]
ONE_X2_CANDIDATES: List[Tuple[str, str, str]] = [
    ("AvgH", "AvgD", "AvgA"),
    ("PSH", "PSD", "PSA"),
    ("B365H", "B365D", "B365A"),
]

FEATURE_COLUMNS = [
    "over_odds_open", "under_odds_open", "fair_over_open", "ou_margin_open",
    "over_odds_close", "under_odds_close", "fair_over_close", "ou_move_to_close",
    "home_odds", "draw_odds", "away_odds",
    "home_prob", "draw_prob", "away_prob", "home_away_prob_gap",
    "home_gf_avg", "home_ga_avg", "away_gf_avg", "away_ga_avg",
    "home_tot_avg", "away_tot_avg", "home_over_rate", "away_over_rate",
    "home_home_gf_avg", "home_home_ga_avg", "away_away_gf_avg", "away_away_ga_avg",
    "home_home_over_rate", "away_away_over_rate",
    "home_gf_last5", "home_ga_last5", "away_gf_last5", "away_ga_last5",
    "home_over_last5", "away_over_last5",
    "expected_goals_proxy", "combined_over_rate",
]


def safe_to_datetime(series: pd.Series) -> pd.Series:
    iso = pd.to_datetime(series, format="%Y-%m-%d", errors="coerce")
    dmy = pd.to_datetime(series, format="%d/%m/%Y", errors="coerce")
    return iso.fillna(dmy)


def first_pair(df: pd.DataFrame, candidates: List[Tuple[str, str]]) -> Tuple[Optional[str], Optional[str]]:
    for over_col, under_col in candidates:
        if over_col in df.columns and under_col in df.columns:
            valid_count = int((df[over_col].notna() & df[under_col].notna()).sum())
            if valid_count > 0:
                return over_col, under_col
    return None, None


def first_triplet(df: pd.DataFrame, candidates: List[Tuple[str, str, str]]) -> Tuple[Optional[str], Optional[str], Optional[str]]:
    for home_col, draw_col, away_col in candidates:
        if all(col in df.columns for col in (home_col, draw_col, away_col)):
            valid_count = int((df[home_col].notna() & df[draw_col].notna() & df[away_col].notna()).sum())
            if valid_count > 0:
                return home_col, draw_col, away_col
    return None, None, None


def mean_or_default(values: List[float], default: float) -> float:
    if not values:
        return float(default)
    return float(np.mean(values))


def extract_prob(odds: float | None) -> float:
    if odds is None or pd.isna(odds) or odds <= 0:
        return np.nan
    return 1.0 / float(odds)


def fair_prob(prob_a: float, prob_b: float) -> float:
    if pd.isna(prob_a) or pd.isna(prob_b) or (prob_a + prob_b) <= 0:
        return np.nan
    return float(prob_a / (prob_a + prob_b))


def load_league_data(league_code: str) -> Optional[pd.DataFrame]:
    """Load historical data for a league and prepare it for training."""
    path = HISTORICAL_DIR / f'{league_code}_history.csv'
    if not path.exists():
        print(f"  [SKIP] {league_code}: file not found")
        return None
    
    df = pd.read_csv(path, low_memory=False)
    df['Date'] = safe_to_datetime(df['Date'])
    df = df.dropna(subset=['Date'])
    df = df.sort_values('Date').reset_index(drop=True)
    
    open_over_col, open_under_col = first_pair(df, OPEN_OU_CANDIDATES)
    close_over_col, close_under_col = first_pair(df, CLOSE_OU_CANDIDATES)
    home_col, draw_col, away_col = first_triplet(df, ONE_X2_CANDIDATES)
    
    if not open_over_col:
        print(f"  [SKIP] {league_code}: no O/U odds columns found")
        return None
    
    print(f"  [OK] {league_code}: {len(df)} rows, open_ou={open_over_col}, close_ou={close_over_col}")
    
    df['league_code'] = league_code
    
    return df


def build_features(df: pd.DataFrame, league_code: str) -> pd.DataFrame:
    """Build features for goal-number prediction."""
    open_over_col, open_under_col = first_pair(df, OPEN_OU_CANDIDATES)
    close_over_col, close_under_col = first_pair(df, CLOSE_OU_CANDIDATES)
    home_col, draw_col, away_col = first_triplet(df, ONE_X2_CANDIDATES)
    
    team_hist = defaultdict(lambda: {
        "gf": [], "ga": [], "tot": [], "over": [],
        "home_gf": [], "home_ga": [], "home_over": [],
        "away_gf": [], "away_ga": [], "away_over": [],
    })
    
    rows = []
    
    for _, match in df.iterrows():
        home = str(match["HomeTeam"])
        away = str(match["AwayTeam"])
        home_hist = team_hist[home]
        away_hist = team_hist[away]
        
        open_over = float(match[open_over_col]) if pd.notna(match[open_over_col]) else np.nan
        open_under = float(match[open_under_col]) if pd.notna(match[open_under_col]) else np.nan
        close_over = float(match[close_over_col]) if close_over_col and pd.notna(match[close_over_col]) else np.nan
        close_under = float(match[close_under_col]) if close_under_col and pd.notna(match[close_under_col]) else np.nan
        
        open_over_prob = extract_prob(open_over)
        open_under_prob = extract_prob(open_under)
        fair_over_open = fair_prob(open_over_prob, open_under_prob)
        
        close_over_prob = extract_prob(close_over)
        close_under_prob = extract_prob(close_under)
        fair_over_close = fair_prob(close_over_prob, close_under_prob)
        
        if home_col:
            home_odds = float(match[home_col]) if pd.notna(match[home_col]) else np.nan
            draw_odds = float(match[draw_col]) if draw_col and pd.notna(match[draw_col]) else np.nan
            away_odds = float(match[away_col]) if away_col and pd.notna(match[away_col]) else np.nan
            home_prob = extract_prob(home_odds)
            draw_prob = extract_prob(draw_odds)
            away_prob = extract_prob(away_odds)
            prob_sum = sum(v for v in [home_prob, draw_prob, away_prob] if not pd.isna(v))
            if prob_sum > 0:
                home_prob = home_prob / prob_sum if not pd.isna(home_prob) else np.nan
                draw_prob = draw_prob / prob_sum if not pd.isna(draw_prob) else np.nan
                away_prob = away_prob / prob_sum if not pd.isna(away_prob) else np.nan
        else:
            home_odds = draw_odds = away_odds = np.nan
            home_prob = draw_prob = away_prob = np.nan
        
        total_goals = float(match.get('FTHG', 0) or 0) + float(match.get('FTAG', 0) or 0)
        
        row = {
            "league_code": league_code,
            "over_odds_open": open_over,
            "under_odds_open": open_under,
            "fair_over_open": fair_over_open,
            "ou_margin_open": (open_over_prob + open_under_prob - 1.0) if not pd.isna(open_over_prob) and not pd.isna(open_under_prob) else np.nan,
            "over_odds_close": close_over,
            "under_odds_close": close_under,
            "fair_over_close": fair_over_close,
            "ou_move_to_close": (fair_over_close - fair_over_open) if not pd.isna(fair_over_close) and not pd.isna(fair_over_open) else np.nan,
            "home_odds": home_odds,
            "draw_odds": draw_odds,
            "away_odds": away_odds,
            "home_prob": home_prob,
            "draw_prob": draw_prob,
            "away_prob": away_prob,
            "home_away_prob_gap": (home_prob - away_prob) if not pd.isna(home_prob) and not pd.isna(away_prob) else np.nan,
            "home_gf_avg": mean_or_default(home_hist["gf"], 1.4),
            "home_ga_avg": mean_or_default(home_hist["ga"], 1.4),
            "away_gf_avg": mean_or_default(away_hist["gf"], 1.2),
            "away_ga_avg": mean_or_default(away_hist["ga"], 1.2),
            "home_tot_avg": mean_or_default(home_hist["tot"], 2.6),
            "away_tot_avg": mean_or_default(away_hist["tot"], 2.6),
            "home_over_rate": mean_or_default(home_hist["over"], 0.5),
            "away_over_rate": mean_or_default(away_hist["over"], 0.5),
            "home_home_gf_avg": mean_or_default(home_hist["home_gf"], mean_or_default(home_hist["gf"], 1.4)),
            "home_home_ga_avg": mean_or_default(home_hist["home_ga"], mean_or_default(home_hist["ga"], 1.4)),
            "away_away_gf_avg": mean_or_default(away_hist["away_gf"], mean_or_default(away_hist["gf"], 1.2)),
            "away_away_ga_avg": mean_or_default(away_hist["away_ga"], mean_or_default(away_hist["ga"], 1.2)),
            "home_home_over_rate": mean_or_default(home_hist["home_over"], mean_or_default(home_hist["over"], 0.5)),
            "away_away_over_rate": mean_or_default(away_hist["away_over"], mean_or_default(away_hist["over"], 0.5)),
            "home_gf_last5": mean_or_default(home_hist["gf"][-5:], mean_or_default(home_hist["gf"], 1.4)),
            "home_ga_last5": mean_or_default(home_hist["ga"][-5:], mean_or_default(home_hist["ga"], 1.4)),
            "away_gf_last5": mean_or_default(away_hist["gf"][-5:], mean_or_default(away_hist["gf"], 1.2)),
            "away_ga_last5": mean_or_default(away_hist["ga"][-5:], mean_or_default(away_hist["ga"], 1.2)),
            "home_over_last5": mean_or_default(home_hist["over"][-5:], mean_or_default(home_hist["over"], 0.5)),
            "away_over_last5": mean_or_default(away_hist["over"][-5:], mean_or_default(away_hist["over"], 0.5)),
            "expected_goals_proxy": float(np.nanmean([
                mean_or_default(home_hist["gf"], 1.4), mean_or_default(home_hist["home_gf"], mean_or_default(home_hist["gf"], 1.4)),
                mean_or_default(away_hist["ga"], 1.2), mean_or_default(away_hist["away_ga"], mean_or_default(away_hist["ga"], 1.2)),
                mean_or_default(away_hist["gf"], 1.2), mean_or_default(away_hist["away_gf"], mean_or_default(away_hist["gf"], 1.2)),
                mean_or_default(home_hist["ga"], 1.4), mean_or_default(home_hist["home_ga"], mean_or_default(home_hist["ga"], 1.4)),
            ])),
            "combined_over_rate": float(np.nanmean([mean_or_default(home_hist["over"], 0.5), mean_or_default(away_hist["over"], 0.5)])),
            "is_over": 1 if total_goals > 2.5 else 0,
        }
        rows.append(row)
        
        # Update team history
        home_hist["gf"].append(total_goals)
        home_hist["ga"].append(total_goals)
        home_hist["tot"].append(total_goals)
        home_hist["over"].append(1 if total_goals > 2.5 else 0)
        if pd.notna(match.get('FTHG')) and pd.notna(match.get('FTAG')):
            home_hist["home_gf"].append(float(match['FTHG']))
            home_hist["home_ga"].append(float(match['FTAG']))
            home_hist["home_over"].append(1 if total_goals > 2.5 else 0)
        
        away_hist["gf"].append(total_goals)
        away_hist["ga"].append(total_goals)
        away_hist["tot"].append(total_goals)
        away_hist["over"].append(1 if total_goals > 2.5 else 0)
        if pd.notna(match.get('FTHG')) and pd.notna(match.get('FTAG')):
            away_hist["away_gf"].append(float(match['FTHG']))
            away_hist["away_ga"].append(float(match['FTAG']))
            away_hist["away_over"].append(1 if total_goals > 2.5 else 0)
    
    return pd.DataFrame(rows)


def train_cross_league_model() -> None:
    """Train a cross-league goal-number model."""
    print("Loading historical data from multiple leagues...")
    
    # Load all leagues with O/U odds
    all_data = []
    for league_code in LEAGUES_WITH_OU:
        df = load_league_data(league_code)
        if df is not None:
            all_data.append(df)
    
    if not all_data:
        print("ERROR: No leagues with O/U odds data found!")
        return
    
    # Combine all data
    print(f"\nCombining data from {len(all_data)} leagues...")
    combined_df = pd.concat(all_data, ignore_index=True)
    print(f"Total rows: {len(combined_df)}")
    
    # Build features
    print("\nBuilding features...")
    feature_dfs = []
    for league_code in combined_df['league_code'].unique():
        league_df = combined_df[combined_df['league_code'] == league_code].copy()
        features_df = build_features(league_df, league_code)
        feature_dfs.append(features_df)
    
    all_features_df = pd.concat(feature_dfs, ignore_index=True)
    print(f"Total feature rows: {len(all_features_df)}")
    
    # Split data
    split_idx = int(len(all_features_df) * 0.8)
    train_df = all_features_df.iloc[:split_idx]
    test_df = all_features_df.iloc[split_idx:]
    
    print(f"Train rows: {len(train_df)}, Test rows: {len(test_df)}")
    
    # Prepare features
    feature_cols = [col for col in FEATURE_COLUMNS if col in all_features_df.columns]
    X_train = train_df[feature_cols].fillna(0)
    y_train = train_df['is_over']
    X_test = test_df[feature_cols].fillna(0)
    y_test = test_df['is_over']
    
    # Train model
    print("\nTraining Random Forest model...")
    pipeline = Pipeline([
        ('imputer', SimpleImputer(strategy='mean')),
        ('scaler', StandardScaler()),
        ('clf', RandomForestClassifier(n_estimators=200, max_depth=10, random_state=42, n_jobs=-1))
    ])
    
    pipeline.fit(X_train, y_train)
    
    # Evaluate
    print("\nEvaluating model...")
    y_pred = pipeline.predict(X_test)
    overall_acc = accuracy_score(y_test, y_pred)
    
    probs = pipeline.predict_proba(X_test)[:, 1]
    
    # Test different thresholds
    thresholds = np.arange(0.50, 0.81, 0.01)
    best_threshold = 0.50
    best_bet_acc = 0.0
    best_bet_count = 0
    
    print("\nThreshold analysis:")
    for threshold in thresholds:
        mask = (probs >= threshold) | (probs <= (1.0 - threshold))
        bet_count = int(mask.sum())
        if bet_count < 50:
            continue
        
        preds = (probs >= threshold).astype(int)
        bet_acc = accuracy_score(y_test[mask], preds[mask])
        
        if bet_acc > best_bet_acc or (bet_acc == best_bet_acc and bet_count > best_bet_count):
            best_bet_acc = bet_acc
            best_bet_count = bet_count
            best_threshold = threshold
        
        if threshold in [0.55, 0.60, 0.65, 0.67, 0.70]:
            print(f"  Threshold={threshold:.2f}: Bet Count={bet_count}, Acc={bet_acc:.2%}")
    
    print(f"\nBest threshold: {best_threshold:.2f}")
    print(f"Best bet accuracy: {best_bet_acc:.2%} (count={best_bet_count})")
    print(f"Overall accuracy: {overall_acc:.2%}")
    
    # Save model
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    model_path = OUTPUT_DIR / 'cross_league_goal_number_pipeline.pkl'
    config_path = OUTPUT_DIR / 'cross_league_goal_number_config.json'
    
    joblib.dump(pipeline, model_path)
    
    config = {
        'model_type': 'RandomForest',
        'threshold': float(best_threshold),
        'bet_count': int(best_bet_count),
        'bet_acc': float(best_bet_acc),
        'overall_acc': float(overall_acc),
        'total_rows': int(len(all_features_df)),
        'train_rows': int(len(train_df)),
        'test_rows': int(len(test_df)),
        'feature_cols': feature_cols,
        'source_leagues': LEAGUES_WITH_OU,
        'target_leagues': TARGET_LEAGUES,
        'task': 'goal-number',
        'target': 'over_under_2_5',
    }
    
    with open(config_path, 'w', encoding='utf-8') as f:
        json.dump(config, f, ensure_ascii=False, indent=2)
    
    print(f"\nModel saved to: {model_path}")
    print(f"Config saved to: {config_path}")
    
    # Copy to goal_number_api_dynamic directory for use
    goal_number_dir = REPO_ROOT / 'artifacts' / 'goal_number_api_dynamic_current'
    goal_number_dir.mkdir(parents=True, exist_ok=True)
    
    shutil.copy(model_path, goal_number_dir / 'goal_number_api_dynamic_pipeline.pkl')
    shutil.copy(config_path, goal_number_dir / 'goal_number_api_dynamic_config.json')
    
    print(f"\nModel copied to goal_number_api_dynamic directory")
    print("Ready for use with target leagues: ", ", ".join(TARGET_LEAGUES))


if __name__ == '__main__':
    train_cross_league_model()
