#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import math
from collections import defaultdict
from pathlib import Path
from typing import Dict, Iterable, List, Tuple

import joblib
import numpy as np
import pandas as pd
from sklearn.ensemble import ExtraTreesClassifier, HistGradientBoostingClassifier, RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

OPEN_OU_CANDIDATES: List[Tuple[str, str]] = [
    ("Avg>2.5", "Avg<2.5"),
    ("P>2.5", "P<2.5"),
    ("B365>2.5", "B365<2.5"),
]
CLOSE_OU_CANDIDATES: List[Tuple[str, str]] = [
    ("AvgC>2.5", "AvgC<2.5"),
    ("PC>2.5", "PC<2.5"),
    ("B365C>2.5", "B365C<2.5"),
]
ONE_X2_CANDIDATES: List[Tuple[str, str, str]] = [
    ("AvgH", "AvgD", "AvgA"),
    ("PSH", "PSD", "PSA"),
    ("B365H", "B365D", "B365A"),
]

LEAGUES = {
    "i1": "意甲", "e0": "英超", "d1": "德甲", "d2": "德乙",
    "sp1": "西甲", "f1": "法甲", "e2": "英冠", "e1": "英甲", "e3": "英乙",
    "p1": "葡超", "tr1": "土超", "br1": "巴甲", "uz1": "乌兹别克",
    "e2n": "荷乙", "sc1": "苏超", "cz1": "捷克甲", "r1": "俄超", "m1": "美职联",
    "j1": "日职联", "a1": "澳超", "f2": "法乙", "g1": "希腊超",
}

FEATURE_COLUMNS = [
    "over_odds_open",
    "under_odds_open",
    "fair_over_open",
    "ou_margin_open",
    "over_odds_close",
    "under_odds_close",
    "fair_over_close",
    "ou_move_to_close",
    "home_odds",
    "draw_odds",
    "away_odds",
    "home_prob",
    "draw_prob",
    "away_prob",
    "home_away_prob_gap",
    "home_gf_avg",
    "home_ga_avg",
    "away_gf_avg",
    "away_ga_avg",
    "home_tot_avg",
    "away_tot_avg",
    "home_over_rate",
    "away_over_rate",
    "home_home_gf_avg",
    "home_home_ga_avg",
    "away_away_gf_avg",
    "away_away_ga_avg",
    "home_home_over_rate",
    "away_away_over_rate",
    "home_gf_last5",
    "home_ga_last5",
    "away_gf_last5",
    "away_ga_last5",
    "home_over_last5",
    "away_over_last5",
    "expected_goals_proxy",
    "combined_over_rate",
]


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Train independent goal-number O/U 2.5 models.")
    parser.add_argument("--data-dir", type=Path, default=Path("historical_data"))
    parser.add_argument("--output-dir", type=Path, default=Path("experiments/goal_number_pass1"))
    parser.add_argument("--min-ou-rows", type=int, default=400)
    parser.add_argument("--min-bets", type=int, default=25)
    parser.add_argument("--holdout-ratio", type=float, default=0.2)
    parser.add_argument("--threshold-start", type=float, default=0.50)
    parser.add_argument("--threshold-stop", type=float, default=0.80)
    parser.add_argument("--threshold-step", type=float, default=0.01)
    return parser.parse_args()


def safe_to_datetime(series: pd.Series) -> pd.Series:
    iso = pd.to_datetime(series, format="%Y-%m-%d", errors="coerce")
    dmy = pd.to_datetime(series, format="%d/%m/%Y", errors="coerce")
    return iso.fillna(dmy)


def first_pair(df: pd.DataFrame, candidates: Iterable[Tuple[str, str]]) -> Tuple[str | None, str | None]:
    for over_col, under_col in candidates:
        if over_col in df.columns and under_col in df.columns:
            valid_count = int((df[over_col].notna() & df[under_col].notna()).sum())
            if valid_count > 0:
                return over_col, under_col
    return None, None


def first_triplet(df: pd.DataFrame, candidates: Iterable[Tuple[str, str, str]]) -> Tuple[str | None, str | None, str | None]:
    for home_col, draw_col, away_col in candidates:
        if all(col in df.columns for col in (home_col, draw_col, away_col)):
            valid_count = int((df[home_col].notna() & df[draw_col].notna() & df[away_col].notna()).sum())
            if valid_count > 0:
                return home_col, draw_col, away_col
    return None, None, None


def mean_or_default(values: List[float], default: float) -> float:
    if not values:
        return float(default)
    return float(np.mean(values))


def extract_prob(odds: float | None) -> float:
    if odds is None or pd.isna(odds) or odds <= 0:
        return np.nan
    return 1.0 / float(odds)


def fair_prob(prob_a: float, prob_b: float) -> float:
    if pd.isna(prob_a) or pd.isna(prob_b) or (prob_a + prob_b) <= 0:
        return np.nan
    return float(prob_a / (prob_a + prob_b))


def scan_ou_coverage(path: Path) -> dict:
    df = pd.read_csv(path, low_memory=False)
    row = {
        "league": path.stem.replace("_history", ""),
        "rows": int(len(df)),
    }
    for over_col, under_col in OPEN_OU_CANDIDATES + CLOSE_OU_CANDIDATES:
        row[f"{over_col}|{under_col}"] = int((df.get(over_col, pd.Series(dtype=float)).notna() & df.get(under_col, pd.Series(dtype=float)).notna()).sum()) if over_col in df.columns and under_col in df.columns else 0
    open_over, open_under = first_pair(df, OPEN_OU_CANDIDATES)
    close_over, close_under = first_pair(df, CLOSE_OU_CANDIDATES)
    home_col, draw_col, away_col = first_triplet(df, ONE_X2_CANDIDATES)
    row["open_ou_source"] = f"{open_over}/{open_under}" if open_over else ""
    row["close_ou_source"] = f"{close_over}/{close_under}" if close_over else ""
    row["one_x2_source"] = f"{home_col}/{draw_col}/{away_col}" if home_col else ""
    row["eligible"] = bool(open_over)
    return row


def build_features(df: pd.DataFrame) -> pd.DataFrame:
    open_over_col, open_under_col = first_pair(df, OPEN_OU_CANDIDATES)
    close_over_col, close_under_col = first_pair(df, CLOSE_OU_CANDIDATES)
    home_odds_col, draw_odds_col, away_odds_col = first_triplet(df, ONE_X2_CANDIDATES)
    if not open_over_col:
        raise ValueError("No usable O/U 2.5 source columns found")

    team_hist = defaultdict(lambda: {
        "gf": [], "ga": [], "tot": [], "over": [],
        "home_gf": [], "home_ga": [], "home_over": [],
        "away_gf": [], "away_ga": [], "away_over": [],
    })
    rows: List[dict] = []

    for _, match in df.iterrows():
        home = str(match["HomeTeam"])
        away = str(match["AwayTeam"])
        home_hist = team_hist[home]
        away_hist = team_hist[away]

        open_over = float(match[open_over_col]) if pd.notna(match[open_over_col]) else np.nan
        open_under = float(match[open_under_col]) if pd.notna(match[open_under_col]) else np.nan
        close_over = float(match[close_over_col]) if close_over_col and pd.notna(match[close_over_col]) else np.nan
        close_under = float(match[close_under_col]) if close_under_col and pd.notna(match[close_under_col]) else np.nan

        open_over_prob = extract_prob(open_over)
        open_under_prob = extract_prob(open_under)
        fair_over_open = fair_prob(open_over_prob, open_under_prob)

        close_over_prob = extract_prob(close_over)
        close_under_prob = extract_prob(close_under)
        fair_over_close = fair_prob(close_over_prob, close_under_prob)

        if home_odds_col:
            home_odds = float(match[home_odds_col]) if pd.notna(match[home_odds_col]) else np.nan
            draw_odds = float(match[draw_odds_col]) if pd.notna(match[draw_odds_col]) else np.nan
            away_odds = float(match[away_odds_col]) if pd.notna(match[away_odds_col]) else np.nan
            home_prob = extract_prob(home_odds)
            draw_prob = extract_prob(draw_odds)
            away_prob = extract_prob(away_odds)
            prob_sum = sum(v for v in [home_prob, draw_prob, away_prob] if not pd.isna(v))
            if prob_sum > 0:
                home_prob = home_prob / prob_sum if not pd.isna(home_prob) else np.nan
                draw_prob = draw_prob / prob_sum if not pd.isna(draw_prob) else np.nan
                away_prob = away_prob / prob_sum if not pd.isna(away_prob) else np.nan
        else:
            home_odds = draw_odds = away_odds = np.nan
            home_prob = draw_prob = away_prob = np.nan

        row = {
            "over_odds_open": open_over,
            "under_odds_open": open_under,
            "fair_over_open": fair_over_open,
            "ou_margin_open": (open_over_prob + open_under_prob - 1.0) if not pd.isna(open_over_prob) and not pd.isna(open_under_prob) else np.nan,
            "over_odds_close": close_over,
            "under_odds_close": close_under,
            "fair_over_close": fair_over_close,
            "ou_move_to_close": (fair_over_close - fair_over_open) if not pd.isna(fair_over_close) and not pd.isna(fair_over_open) else np.nan,
            "home_odds": home_odds,
            "draw_odds": draw_odds,
            "away_odds": away_odds,
            "home_prob": home_prob,
            "draw_prob": draw_prob,
            "away_prob": away_prob,
            "home_away_prob_gap": (home_prob - away_prob) if not pd.isna(home_prob) and not pd.isna(away_prob) else np.nan,
            "home_gf_avg": mean_or_default(home_hist["gf"], 1.4),
            "home_ga_avg": mean_or_default(home_hist["ga"], 1.4),
            "away_gf_avg": mean_or_default(away_hist["gf"], 1.2),
            "away_ga_avg": mean_or_default(away_hist["ga"], 1.2),
            "home_tot_avg": mean_or_default(home_hist["tot"], 2.6),
            "away_tot_avg": mean_or_default(away_hist["tot"], 2.6),
            "home_over_rate": mean_or_default(home_hist["over"], 0.5),
            "away_over_rate": mean_or_default(away_hist["over"], 0.5),
            "home_home_gf_avg": mean_or_default(home_hist["home_gf"], mean_or_default(home_hist["gf"], 1.4)),
            "home_home_ga_avg": mean_or_default(home_hist["home_ga"], mean_or_default(home_hist["ga"], 1.4)),
            "away_away_gf_avg": mean_or_default(away_hist["away_gf"], mean_or_default(away_hist["gf"], 1.2)),
            "away_away_ga_avg": mean_or_default(away_hist["away_ga"], mean_or_default(away_hist["ga"], 1.2)),
            "home_home_over_rate": mean_or_default(home_hist["home_over"], mean_or_default(home_hist["over"], 0.5)),
            "away_away_over_rate": mean_or_default(away_hist["away_over"], mean_or_default(away_hist["over"], 0.5)),
            "home_gf_last5": mean_or_default(home_hist["gf"][-5:], mean_or_default(home_hist["gf"], 1.4)),
            "home_ga_last5": mean_or_default(home_hist["ga"][-5:], mean_or_default(home_hist["ga"], 1.4)),
            "away_gf_last5": mean_or_default(away_hist["gf"][-5:], mean_or_default(away_hist["gf"], 1.2)),
            "away_ga_last5": mean_or_default(away_hist["ga"][-5:], mean_or_default(away_hist["ga"], 1.2)),
            "home_over_last5": mean_or_default(home_hist["over"][-5:], mean_or_default(home_hist["over"], 0.5)),
            "away_over_last5": mean_or_default(away_hist["over"][-5:], mean_or_default(away_hist["over"], 0.5)),
        }
        row["expected_goals_proxy"] = float(np.nanmean([
            row["home_gf_avg"], row["home_home_gf_avg"], row["away_ga_avg"], row["away_away_ga_avg"],
            row["away_gf_avg"], row["away_away_gf_avg"], row["home_ga_avg"], row["home_home_ga_avg"],
        ]))
        row["combined_over_rate"] = float(np.nanmean([
            row["home_over_rate"], row["away_over_rate"], row["home_over_last5"], row["away_over_last5"],
        ]))
        rows.append(row)

        home_goals = float(match["FTHG"])
        away_goals = float(match["FTAG"])
        total_goals = home_goals + away_goals
        is_over = 1.0 if total_goals > 2.5 else 0.0

        home_hist["gf"].append(home_goals)
        home_hist["ga"].append(away_goals)
        home_hist["tot"].append(total_goals)
        home_hist["over"].append(is_over)
        home_hist["home_gf"].append(home_goals)
        home_hist["home_ga"].append(away_goals)
        home_hist["home_over"].append(is_over)

        away_hist["gf"].append(away_goals)
        away_hist["ga"].append(home_goals)
        away_hist["tot"].append(total_goals)
        away_hist["over"].append(is_over)
        away_hist["away_gf"].append(away_goals)
        away_hist["away_ga"].append(home_goals)
        away_hist["away_over"].append(is_over)

    return pd.DataFrame(rows, columns=FEATURE_COLUMNS)


def build_models() -> Dict[str, Pipeline]:
    return {
        "LR": Pipeline([
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", StandardScaler()),
            ("clf", LogisticRegression(max_iter=2000, class_weight="balanced", C=0.5)),
        ]),
        "RF": Pipeline([
            ("imputer", SimpleImputer(strategy="median")),
            ("clf", RandomForestClassifier(
                n_estimators=400,
                max_depth=6,
                min_samples_leaf=8,
                random_state=42,
                class_weight="balanced_subsample",
            )),
        ]),
        "ET": Pipeline([
            ("imputer", SimpleImputer(strategy="median")),
            ("clf", ExtraTreesClassifier(
                n_estimators=500,
                max_depth=7,
                min_samples_leaf=6,
                random_state=42,
                class_weight="balanced",
            )),
        ]),
        "HGB": Pipeline([
            ("imputer", SimpleImputer(strategy="median")),
            ("clf", HistGradientBoostingClassifier(
                max_depth=5,
                learning_rate=0.05,
                max_iter=300,
                random_state=42,
            )),
        ]),
    }


def threshold_grid(start: float, stop: float, step: float) -> List[float]:
    values: List[float] = []
    x = start
    while x <= stop + 1e-9:
        values.append(round(float(x), 4))
        x += step
    return values


def prepare_league_df(path: Path) -> tuple[pd.DataFrame, dict]:
    df = pd.read_csv(path, low_memory=False)
    if "Date" not in df.columns:
        raise ValueError("missing Date column")
    for col in ["HomeTeam", "AwayTeam", "FTHG", "FTAG"]:
        if col not in df.columns:
            raise ValueError(f"missing {col} column")

    df = df.copy()
    df["date_obj"] = safe_to_datetime(df["Date"])
    df = df.dropna(subset=["date_obj", "HomeTeam", "AwayTeam", "FTHG", "FTAG"]).sort_values("date_obj").reset_index(drop=True)

    open_over_col, open_under_col = first_pair(df, OPEN_OU_CANDIDATES)
    close_over_col, close_under_col = first_pair(df, CLOSE_OU_CANDIDATES)
    home_odds_col, draw_odds_col, away_odds_col = first_triplet(df, ONE_X2_CANDIDATES)
    if not open_over_col:
        raise ValueError("missing usable O/U 2.5 columns")

    df = df[df[open_over_col].notna() & df[open_under_col].notna()].reset_index(drop=True)
    meta = {
        "open_ou_source": f"{open_over_col}/{open_under_col}",
        "close_ou_source": f"{close_over_col}/{close_under_col}" if close_over_col else "",
        "one_x2_source": f"{home_odds_col}/{draw_odds_col}/{away_odds_col}" if home_odds_col else "",
        "ou_rows": int(len(df)),
    }
    return df, meta


def train_one_league(
    league_code: str,
    league_name: str,
    data_path: Path,
    output_dir: Path,
    min_bets: int,
    holdout_ratio: float,
    thresholds: List[float],
) -> tuple[dict, pd.DataFrame]:
    df, meta = prepare_league_df(data_path)
    if len(df) < 200:
        raise ValueError(f"not enough usable rows after O/U filter: {len(df)}")

    X = build_features(df)
    y = ((df["FTHG"].astype(float) + df["FTAG"].astype(float)) > 2.5).astype(int)

    split_idx = int(len(df) * (1.0 - holdout_ratio))
    if split_idx < 150 or (len(df) - split_idx) < 50:
        raise ValueError(f"train/test split too small for {league_code}: {len(df)} rows")

    X_train, X_test = X.iloc[:split_idx], X.iloc[split_idx:]
    y_train, y_test = y.iloc[:split_idx], y.iloc[split_idx:]

    detailed_rows: List[dict] = []
    best = None
    best_pipeline = None

    for model_name, pipeline in build_models().items():
        pipeline.fit(X_train, y_train)
        probs = pipeline.predict_proba(X_test)[:, 1]
        preds = (probs >= 0.5).astype(int)
        overall_acc = float(accuracy_score(y_test, preds))

        for threshold in thresholds:
            mask = (probs >= threshold) | (probs <= (1.0 - threshold))
            bet_count = int(mask.sum())
            if bet_count < min_bets:
                continue
            bet_acc = float(accuracy_score(y_test[mask], preds[mask]))
            row = {
                "league": league_code,
                "league_name": league_name,
                "model": model_name,
                "threshold": float(threshold),
                "bet_count": bet_count,
                "bet_acc": bet_acc,
                "overall_acc": overall_acc,
                "rows": int(len(df)),
                "train_rows": int(len(X_train)),
                "test_rows": int(len(X_test)),
                "open_ou_source": meta["open_ou_source"],
                "close_ou_source": meta["close_ou_source"],
                "one_x2_source": meta["one_x2_source"],
            }
            detailed_rows.append(row)
            if best is None or bet_acc > best["bet_acc"] or (bet_acc == best["bet_acc"] and bet_count > best["bet_count"]):
                best = row
                best_pipeline = pipeline

    if best is None or best_pipeline is None:
        raise ValueError(f"no model met min_bets={min_bets}")

    model_dir = output_dir / "models" / league_code
    model_dir.mkdir(parents=True, exist_ok=True)
    model_path = model_dir / "goal_number_pipeline.pkl"
    config_path = model_dir / "goal_number_config.json"

    config = {
        **best,
        "task": "goal-number",
        "target": "over_under_2_5",
        "features": FEATURE_COLUMNS,
        "data_path": str(data_path),
        "model_path": str(model_path),
    }

    joblib.dump(best_pipeline, model_path)
    config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8")
    return config, pd.DataFrame(detailed_rows)


def main() -> None:
    args = parse_args()
    data_dir = args.data_dir
    output_dir = args.output_dir
    output_dir.mkdir(parents=True, exist_ok=True)

    coverage_rows: List[dict] = []
    for path in sorted(data_dir.glob("*_history.csv")):
        coverage_rows.append(scan_ou_coverage(path))
    coverage_df = pd.DataFrame(coverage_rows).sort_values(["eligible", "rows"], ascending=[False, False])
    coverage_df.to_csv(output_dir / "ou_league_coverage.csv", index=False, encoding="utf-8-sig")

    thresholds = threshold_grid(args.threshold_start, args.threshold_stop, args.threshold_step)
    summary_rows: List[dict] = []
    detailed_frames: List[pd.DataFrame] = []
    errors: List[dict] = []

    for coverage_row in coverage_rows:
        league_code = coverage_row["league"]
        data_path = data_dir / f"{league_code}_history.csv"
        if not coverage_row["eligible"]:
            errors.append({"league": league_code, "error": "no usable OU columns"})
            continue
        if not coverage_row["open_ou_source"]:
            errors.append({"league": league_code, "error": "no open OU source"})
            continue

        open_ou_key = coverage_row["open_ou_source"].replace("/", "|")
        ou_rows = int(coverage_row.get(open_ou_key, 0))
        if ou_rows < args.min_ou_rows:
            errors.append({"league": league_code, "error": f"OU rows {ou_rows} < min-ou-rows {args.min_ou_rows}"})
            continue

        try:
            config, details_df = train_one_league(
                league_code=league_code,
                league_name=LEAGUES.get(league_code, league_code),
                data_path=data_path,
                output_dir=output_dir,
                min_bets=args.min_bets,
                holdout_ratio=args.holdout_ratio,
                thresholds=thresholds,
            )
            summary_rows.append(config)
            detailed_frames.append(details_df)
            print(f"[OK] {league_code}: {config['model']} thr={config['threshold']:.2f} bet_acc={config['bet_acc']:.4f} bets={config['bet_count']}")
        except Exception as exc:  # noqa: BLE001
            errors.append({"league": league_code, "error": str(exc)})
            print(f"[FAIL] {league_code}: {exc}")

    summary_df = pd.DataFrame(summary_rows).sort_values(["bet_acc", "bet_count"], ascending=[False, False]) if summary_rows else pd.DataFrame()
    detailed_df = pd.concat(detailed_frames, ignore_index=True) if detailed_frames else pd.DataFrame()
    errors_df = pd.DataFrame(errors)

    if not summary_df.empty:
        summary_df.to_csv(output_dir / "best_results.csv", index=False, encoding="utf-8-sig")
    if not detailed_df.empty:
        detailed_df.to_csv(output_dir / "all_results.csv", index=False, encoding="utf-8-sig")
    if not errors_df.empty:
        errors_df.to_csv(output_dir / "skipped_or_failed.csv", index=False, encoding="utf-8-sig")

    passed_df = summary_df[summary_df["bet_acc"] >= 0.75].copy() if not summary_df.empty else pd.DataFrame()
    combined = {}
    if not passed_df.empty:
        combined_correct = int(np.round((passed_df["bet_acc"] * passed_df["bet_count"]).sum()))
        combined_bets = int(passed_df["bet_count"].sum())
        combined = {
            "task": "goal-number",
            "target": "over_under_2_5",
            "qualified_leagues": passed_df["league"].tolist(),
            "qualified_count": int(len(passed_df)),
            "combined_bets": combined_bets,
            "combined_bet_acc": float(combined_correct / combined_bets) if combined_bets else 0.0,
        }

    summary = {
        "task": "goal-number",
        "target": "over_under_2_5",
        "min_ou_rows": args.min_ou_rows,
        "min_bets": args.min_bets,
        "holdout_ratio": args.holdout_ratio,
        "eligible_leagues": coverage_df.loc[coverage_df["eligible"], "league"].tolist(),
        "trained_leagues": summary_df["league"].tolist() if not summary_df.empty else [],
        "combined_passed": combined,
        "errors": errors,
    }
    (output_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(summary, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()
