#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import math
import os
import re
import sys
import time
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Tuple

import joblib
import numpy as np
import pandas as pd
import requests
from sklearn.ensemble import ExtraTreesClassifier, HistGradientBoostingClassifier, RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

from team_name_zh import ensure_team_name_map

REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT / 'scripts') not in sys.path:
    sys.path.insert(0, str(REPO_ROOT / 'scripts'))

BEIJING_TZ = timezone(timedelta(hours=8))
ARTIFACT_DIR = REPO_ROOT / 'artifacts' / 'goal_number_api_dynamic_current'
CACHE_DIR = REPO_ROOT / 'artifacts' / 'goal_number_api_dynamic_cache'
PREDICTIONS_DIR = REPO_ROOT / 'predictions'
SUPPORTED_FINAL_STATUSES = {'FT', 'AET', 'PEN'}

LEAGUE_CODE_MAP: Dict[Tuple[str, str], str] = {
    ('Australia', 'A-League'): 'a1',
    ('Brazil', 'Serie A'): 'br1',
    ('Czech-Republic', 'Czech Liga'): 'cz1',
    ('Germany', 'Bundesliga'): 'd1',
    ('Germany', '2. Bundesliga'): 'd2',
    ('England', 'Premier League'): 'e0',
    ('England', 'Championship'): 'e2',
    ('England', 'League One'): 'e1',
    ('England', 'League Two'): 'e3',
    ('France', 'Ligue 1'): 'f1',
    ('France', 'Ligue 2'): 'f2',
    ('Greece', 'Super League 1'): 'g1',
    ('Italy', 'Serie A'): 'i1',
    ('Japan', 'J1 League'): 'j1',
    ('Japan', 'J2/J3 League'): 'j2',
    ('Netherlands', 'Eerste Divisie'): 'e2n',
    ('Portugal', 'Primeira Liga'): 'p1',
    ('Russia', 'Premier League'): 'r1',
    ('Scotland', 'Premiership'): 'sc1',
    ('South-Korea', 'K League 2'): 'k2',
    ('Spain', 'La Liga'): 'sp1',
    ('Turkey', 'Süper Lig'): 'tr1',
    ('USA', 'Major League Soccer'): 'm1',
    ('Uzbekistan', 'Super League'): 'uz1',
}

LEAGUE_ZH_MAP: Dict[str, str] = {
    'a1': '澳超',
    'br1': '巴甲',
    'cz1': '捷克甲',
    'd1': '德甲',
    'd2': '德乙',
    'e0': '英超',
    'e1': '英甲',
    'e2': '英冠',
    'e3': '英乙',
    'e2n': '荷乙',
    'f1': '法甲',
    'f2': '法乙',
    'g1': '希腊超',
    'i1': '意甲',
    'j1': '日职联',
    'j2': '日职乙',
    'k2': '韩国K2',
    'm1': '美职联',
    'p1': '葡超',
    'r1': '俄超',
    'sc1': '苏超',
    'sp1': '西甲',
    'tr1': '土超',
    'uz1': '乌兹别克超',
}

FEATURE_COLUMNS = [
    'ou_line', 'over_odds', 'under_odds', 'fair_over_open', 'ou_margin_open', 'ou_quote_count',
    'home_odds', 'draw_odds', 'away_odds', 'home_prob', 'draw_prob', 'away_prob', 'home_away_prob_gap',
    'home_gf_avg', 'home_ga_avg', 'away_gf_avg', 'away_ga_avg', 'home_tot_avg', 'away_tot_avg',
    'home_over_rate', 'away_over_rate', 'home_gf_last5', 'home_ga_last5', 'away_gf_last5', 'away_ga_last5',
    'home_over_last5', 'away_over_last5', 'home_home_gf_avg', 'home_home_ga_avg', 'away_away_gf_avg',
    'away_away_ga_avg', 'home_home_over_rate', 'away_away_over_rate', 'expected_goals_proxy',
    'combined_over_rate', 'league_tot_avg', 'league_over_rate', 'league_match_count',
    'home_match_count', 'away_match_count', 'days_since_home_last', 'days_since_away_last',
    'line_minus_expected_goals', 'line_minus_league_avg', 'fair_over_minus_even',
]


@dataclass
class TrainArtifacts:
    model: Pipeline
    summary: dict
    config: dict
    model_path: Path
    summary_path: Path
    config_path: Path


class APIFootballClient:
    def __init__(
        self,
        base_url: str,
        api_key: str,
        cache_dir: Path = CACHE_DIR,
        force_refresh: bool = False,
        sleep_seconds: float = 0.0,
    ) -> None:
        self.base_url = base_url.rstrip('/')
        self.cache_dir = cache_dir
        self.force_refresh = force_refresh
        self.sleep_seconds = sleep_seconds
        self.session = requests.Session()
        self.session.headers.update({'x-apisports-key': api_key})

    def _cache_path(self, kind: str, key: str) -> Path:
        safe = re.sub(r'[^A-Za-z0-9_.=-]+', '_', key)
        return self.cache_dir / f'{kind}_{safe}.json'

    def _read_cache(self, path: Path) -> Any | None:
        if path.exists() and not self.force_refresh:
            return json.loads(path.read_text(encoding='utf-8'))
        return None

    def _write_cache(self, path: Path, payload: Any) -> Any:
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(json.dumps(payload, ensure_ascii=False), encoding='utf-8')
        return payload

    def get_json(self, endpoint: str, params: dict) -> dict:
        if self.sleep_seconds > 0:
            time.sleep(self.sleep_seconds)
        resp = self.session.get(self.base_url + endpoint, params=params, timeout=90)
        resp.raise_for_status()
        return resp.json()

    def odds_date_meta(self, date_str: str) -> dict:
        path = self._cache_path('odds_meta', date_str)
        cached = self._read_cache(path)
        if cached is not None:
            return cached
        payload = self.get_json('/odds', {'date': date_str, 'page': 1})
        meta = {'results': payload.get('results', 0), 'paging': payload.get('paging', {}), 'errors': payload.get('errors', [])}
        return self._write_cache(path, meta)

    def odds_by_date(self, date_str: str) -> List[dict]:
        path = self._cache_path('odds_date', date_str)
        cached = self._read_cache(path)
        if cached is not None:
            return cached
        page = 1
        total = 1
        rows: List[dict] = []
        while page <= total:
            payload = self.get_json('/odds', {'date': date_str, 'page': page})
            total = int((payload.get('paging') or {}).get('total', 1) or 1)
            rows.extend(payload.get('response', []))
            page += 1
        return self._write_cache(path, rows)

    def fixtures_by_date(self, date_str: str) -> List[dict]:
        path = self._cache_path('fixtures_date', date_str)
        cached = self._read_cache(path)
        if cached is not None:
            return cached
        payload = self.get_json('/fixtures', {'date': date_str})
        rows = payload.get('response', [])
        return self._write_cache(path, rows)

    def fixtures_by_league_season(self, league_id: int, season: int) -> List[dict]:
        key = f'{league_id}_{season}'
        path = self._cache_path('fixtures_league_season', key)
        cached = self._read_cache(path)
        if cached is not None:
            return cached
        payload = self.get_json('/fixtures', {'league': league_id, 'season': season})
        rows = payload.get('response', [])
        return self._write_cache(path, rows)


def load_serving_codes() -> set[str]:
    manifest_path = REPO_ROOT / 'artifacts' / 'v4_models_serving_current' / 'manifest.json'
    if not manifest_path.exists():
        return set(LEAGUE_ZH_MAP.keys())
    try:
        payload = json.loads(manifest_path.read_text(encoding='utf-8'))
        if isinstance(payload, list):
            return {str(item.get('code')) for item in payload if item.get('code')}
    except Exception:
        pass
    return set(LEAGUE_ZH_MAP.keys())


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description='API-Football dynamic goal-number trainer + predictor.')
    parser.add_argument('--cache-dir', type=Path, default=CACHE_DIR)
    parser.add_argument('--artifact-dir', type=Path, default=ARTIFACT_DIR)
    parser.add_argument('--predictions-dir', type=Path, default=PREDICTIONS_DIR)
    parser.add_argument('--force-refresh', action='store_true')
    parser.add_argument('--api-sleep-seconds', type=float, default=0.0)

    sub = parser.add_subparsers(dest='command', required=True)

    train = sub.add_parser('train', help='Train dynamic goal-number model from recent API-Football odds window')
    train.add_argument('--end-date', help='YYYY-MM-DD, defaults to Beijing yesterday')
    train.add_argument('--max-lookback-days', type=int, default=21)
    train.add_argument('--min-bets', type=int, default=25)
    train.add_argument('--min-train-rows', type=int, default=300)
    train.add_argument('--threshold-start', type=float, default=0.50)
    train.add_argument('--threshold-stop', type=float, default=0.80)
    train.add_argument('--threshold-step', type=float, default=0.01)

    predict = sub.add_parser('predict', help='Generate next-N-hour predictions from trained model')
    predict.add_argument('--hours', type=int, default=24)
    predict.add_argument('--now', help='Override current Beijing time, format YYYY-MM-DD HH:MM:SS')

    refresh = sub.add_parser('refresh', help='Train then predict in one pass')
    refresh.add_argument('--end-date', help='YYYY-MM-DD, defaults to Beijing yesterday')
    refresh.add_argument('--max-lookback-days', type=int, default=21)
    refresh.add_argument('--min-bets', type=int, default=25)
    refresh.add_argument('--min-train-rows', type=int, default=300)
    refresh.add_argument('--threshold-start', type=float, default=0.50)
    refresh.add_argument('--threshold-stop', type=float, default=0.80)
    refresh.add_argument('--threshold-step', type=float, default=0.01)
    refresh.add_argument('--hours', type=int, default=24)
    refresh.add_argument('--now', help='Override current Beijing time, format YYYY-MM-DD HH:MM:SS')

    return parser.parse_args()


def now_beijing() -> datetime:
    return datetime.now(timezone.utc).astimezone(BEIJING_TZ)


def parse_now_beijing(raw: str | None) -> datetime:
    if not raw:
        return now_beijing()
    dt = datetime.strptime(raw, '%Y-%m-%d %H:%M:%S')
    return dt.replace(tzinfo=BEIJING_TZ)


def default_train_end_date() -> str:
    return (now_beijing().date() - timedelta(days=1)).isoformat()


def mean_or_default(values: List[float], default: float) -> float:
    return float(np.mean(values)) if values else float(default)


def extract_prob(odds: float | None) -> float:
    if odds is None or pd.isna(odds) or float(odds) <= 0:
        return np.nan
    return 1.0 / float(odds)


def normalize_team(name: str | None) -> str:
    return re.sub(r'\s+', ' ', (name or '').strip())


def parse_total_value(raw: str | None) -> Optional[Tuple[str, float]]:
    m = re.match(r'^(Over|Under)\s+([0-9]+(?:\.[0-9]+)?)$', str(raw or '').strip(), re.I)
    if not m:
        return None
    return m.group(1).lower(), float(m.group(2))


def choose_main_total(bookmakers: List[dict]) -> Optional[dict]:
    per_line: Dict[float, dict] = {}
    for bookmaker in bookmakers:
        for bet in bookmaker.get('bets', []):
            if bet.get('name') != 'Goals Over/Under':
                continue
            local: Dict[float, dict] = defaultdict(dict)
            for item in bet.get('values', []):
                parsed = parse_total_value(item.get('value'))
                if not parsed:
                    continue
                side, line = parsed
                try:
                    odd = float(item.get('odd'))
                except Exception:
                    continue
                if odd <= 1:
                    continue
                local[line][side] = odd
            for line, sides in local.items():
                if 'over' not in sides or 'under' not in sides:
                    continue
                rec = per_line.setdefault(line, {'quotes': 0, 'over': [], 'under': []})
                rec['quotes'] += 1
                rec['over'].append(float(sides['over']))
                rec['under'].append(float(sides['under']))
    candidates: List[dict] = []
    for line, rec in per_line.items():
        over_odds = float(np.mean(rec['over']))
        under_odds = float(np.mean(rec['under']))
        over_prob = extract_prob(over_odds)
        under_prob = extract_prob(under_odds)
        fair_over = over_prob / (over_prob + under_prob) if not pd.isna(over_prob) and not pd.isna(under_prob) and (over_prob + under_prob) > 0 else np.nan
        candidates.append({
            'ou_line': float(line),
            'over_odds': over_odds,
            'under_odds': under_odds,
            'ou_quote_count': int(rec['quotes']),
            'fair_over_open': fair_over,
            'even_gap': abs((fair_over if not pd.isna(fair_over) else 0.5) - 0.5),
        })
    if not candidates:
        return None
    candidates.sort(key=lambda x: (-x['ou_quote_count'], x['even_gap'], abs(x['ou_line'] - 2.5)))
    chosen = candidates[0].copy()
    chosen.pop('even_gap', None)
    return chosen


def choose_main_1x2(bookmakers: List[dict]) -> dict:
    rows = []
    for bookmaker in bookmakers:
        for bet in bookmaker.get('bets', []):
            if bet.get('name') != 'Match Winner':
                continue
            values = {v.get('value'): v.get('odd') for v in bet.get('values', [])}
            try:
                home = float(values['Home'])
                draw = float(values['Draw'])
                away = float(values['Away'])
            except Exception:
                continue
            if min(home, draw, away) <= 1:
                continue
            rows.append((home, draw, away))
    if not rows:
        return {'home_odds': np.nan, 'draw_odds': np.nan, 'away_odds': np.nan}
    arr = np.array(rows, dtype=float)
    return {
        'home_odds': float(arr[:, 0].mean()),
        'draw_odds': float(arr[:, 1].mean()),
        'away_odds': float(arr[:, 2].mean()),
    }


def settle_binary_full_only(total_goals: int, line: float) -> Optional[int]:
    frac = round(line - math.floor(line), 2)
    base = math.floor(line)
    if frac == 0.0:
        if total_goals > line:
            return 1
        if total_goals < line:
            return 0
        return None
    if frac == 0.5:
        return 1 if total_goals > line else 0
    if frac == 0.25:
        if total_goals >= base + 1:
            return 1
        if total_goals <= base - 1:
            return 0
        return None
    if frac == 0.75:
        if total_goals >= base + 2:
            return 1
        if total_goals <= base:
            return 0
        return None
    return None


def threshold_grid(start: float, stop: float, step: float) -> List[float]:
    vals: List[float] = []
    x = start
    while x <= stop + 1e-9:
        vals.append(round(float(x), 4))
        x += step
    return vals


def detect_recent_odds_dates(client: APIFootballClient, end_date: str, max_lookback_days: int) -> List[str]:
    end = datetime.strptime(end_date, '%Y-%m-%d').date()
    found: List[str] = []
    seen_positive = False
    for offset in range(max_lookback_days + 1):
        day = end - timedelta(days=offset)
        date_str = day.isoformat()
        meta = client.odds_date_meta(date_str)
        has_rows = int(meta.get('results', 0) or 0) > 0
        if has_rows:
            found.append(date_str)
            seen_positive = True
        elif seen_positive:
            break
    return sorted(found)


def fixture_kickoff_dt(item: dict) -> Optional[datetime]:
    raw = ((item or {}).get('fixture') or {}).get('date')
    if not raw:
        return None
    return pd.to_datetime(raw, utc=True, errors='coerce').to_pydatetime() if pd.notna(pd.to_datetime(raw, utc=True, errors='coerce')) else None


def collect_completed_target_rows(client: APIFootballClient, dates: List[str], allowed_codes: set[str]) -> pd.DataFrame:
    rows: List[dict] = []
    for date_str in dates:
        fixtures = client.fixtures_by_date(date_str)
        fixture_map = {int(x['fixture']['id']): x for x in fixtures}
        odds_rows = client.odds_by_date(date_str)
        keep_count = 0
        for item in odds_rows:
            fixture_id = int(item['fixture']['id'])
            fx = fixture_map.get(fixture_id)
            if not fx:
                continue
            if fx['fixture']['status']['short'] not in SUPPORTED_FINAL_STATUSES:
                continue
            chosen_total = choose_main_total(item.get('bookmakers', []))
            if not chosen_total:
                continue
            chosen_1x2 = choose_main_1x2(item.get('bookmakers', []))
            kickoff = fixture_kickoff_dt(fx)
            if kickoff is None:
                continue
            league = fx.get('league', {})
            league_code = LEAGUE_CODE_MAP.get((league.get('country'), league.get('name')), f"api_{league.get('id')}")
            if league_code not in allowed_codes:
                continue
            rows.append({
                'fixture_id': fixture_id,
                'kickoff_dt': kickoff.isoformat(),
                'Date': kickoff.astimezone(BEIJING_TZ).strftime('%Y-%m-%d'),
                'country': league.get('country'),
                'league_name': league.get('name'),
                'league_id': int(league.get('id')),
                'season': int(league.get('season')),
                'league_code': league_code,
                'HomeTeam': normalize_team(fx['teams']['home']['name']),
                'AwayTeam': normalize_team(fx['teams']['away']['name']),
                'FTHG': float(fx['goals']['home']),
                'FTAG': float(fx['goals']['away']),
                **chosen_total,
                **chosen_1x2,
            })
            keep_count += 1
        print(f'[collect_train] {date_str}: {keep_count} completed rows')
    df = pd.DataFrame(rows)
    if df.empty:
        return df
    df['kickoff_dt'] = pd.to_datetime(df['kickoff_dt'], utc=True)
    return df.sort_values(['kickoff_dt', 'fixture_id']).reset_index(drop=True)


def collect_history_rows(client: APIFootballClient, league_seasons: Iterable[Tuple[int, int]], allowed_codes: set[str]) -> pd.DataFrame:
    rows: List[dict] = []
    seen = set()
    for league_id, season in sorted(set((int(a), int(b)) for a, b in league_seasons)):
        payload = client.fixtures_by_league_season(league_id, season)
        final_rows = 0
        for item in payload:
            fixture_id = int(item['fixture']['id'])
            if fixture_id in seen:
                continue
            if item['fixture']['status']['short'] not in SUPPORTED_FINAL_STATUSES:
                continue
            kickoff = fixture_kickoff_dt(item)
            if kickoff is None:
                continue
            seen.add(fixture_id)
            league = item.get('league', {})
            league_code = LEAGUE_CODE_MAP.get((league.get('country'), league.get('name')), f"api_{league.get('id')}")
            if league_code not in allowed_codes:
                continue
            rows.append({
                'fixture_id': fixture_id,
                'kickoff_dt': kickoff.isoformat(),
                'Date': kickoff.astimezone(BEIJING_TZ).strftime('%Y-%m-%d'),
                'country': league.get('country'),
                'league_name': league.get('name'),
                'league_id': int(league.get('id')),
                'season': int(league.get('season')),
                'league_code': league_code,
                'HomeTeam': normalize_team(item['teams']['home']['name']),
                'AwayTeam': normalize_team(item['teams']['away']['name']),
                'FTHG': float(item['goals']['home']),
                'FTAG': float(item['goals']['away']),
                'ou_line': np.nan,
                'over_odds': np.nan,
                'under_odds': np.nan,
                'ou_quote_count': np.nan,
                'fair_over_open': np.nan,
                'home_odds': np.nan,
                'draw_odds': np.nan,
                'away_odds': np.nan,
            })
            final_rows += 1
        print(f'[collect_history] league={league_id} season={season}: {final_rows} final fixtures')
    df = pd.DataFrame(rows)
    if df.empty:
        return df
    df['kickoff_dt'] = pd.to_datetime(df['kickoff_dt'], utc=True)
    return df.sort_values(['kickoff_dt', 'fixture_id']).reset_index(drop=True)


def merge_history_with_targets(history_df: pd.DataFrame, target_df: pd.DataFrame) -> pd.DataFrame:
    if history_df.empty:
        return target_df.copy()
    if target_df.empty:
        return history_df.copy()
    keep_cols = [
        'fixture_id', 'ou_line', 'over_odds', 'under_odds', 'ou_quote_count', 'fair_over_open',
        'home_odds', 'draw_odds', 'away_odds',
    ]
    merged = history_df.merge(target_df[keep_cols], on='fixture_id', how='left', suffixes=('', '_target'))
    for col in ['ou_line', 'over_odds', 'under_odds', 'ou_quote_count', 'fair_over_open', 'home_odds', 'draw_odds', 'away_odds']:
        if f'{col}_target' in merged.columns:
            merged[col] = merged[f'{col}_target'].combine_first(merged[col])
            merged = merged.drop(columns=[f'{col}_target'])
    merged['has_market'] = merged['ou_line'].notna().astype(int)
    return merged.sort_values(['kickoff_dt', 'fixture_id']).reset_index(drop=True)


def build_feature_frame(all_rows: pd.DataFrame) -> pd.DataFrame:
    team_hist = defaultdict(lambda: {
        'gf': [], 'ga': [], 'tot': [], 'over': [],
        'home_gf': [], 'home_ga': [], 'home_over': [],
        'away_gf': [], 'away_ga': [], 'away_over': [],
        'last_dt': None,
    })
    league_hist = defaultdict(lambda: {'tot': [], 'over': []})
    features: List[dict] = []

    for _, match in all_rows.iterrows():
        league_key = (int(match['league_id']), int(match['season']))
        home_key = (int(match['league_id']), int(match['season']), str(match['HomeTeam']))
        away_key = (int(match['league_id']), int(match['season']), str(match['AwayTeam']))
        hh = team_hist[home_key]
        aa = team_hist[away_key]
        ll = league_hist[league_key]

        over_prob = extract_prob(match.get('over_odds'))
        under_prob = extract_prob(match.get('under_odds'))
        fair_over = over_prob / (over_prob + under_prob) if not pd.isna(over_prob) and not pd.isna(under_prob) and (over_prob + under_prob) > 0 else np.nan

        home_odds = float(match['home_odds']) if pd.notna(match.get('home_odds')) else np.nan
        draw_odds = float(match['draw_odds']) if pd.notna(match.get('draw_odds')) else np.nan
        away_odds = float(match['away_odds']) if pd.notna(match.get('away_odds')) else np.nan
        home_prob = extract_prob(home_odds)
        draw_prob = extract_prob(draw_odds)
        away_prob = extract_prob(away_odds)
        prob_sum = sum(v for v in [home_prob, draw_prob, away_prob] if not pd.isna(v))
        if prob_sum > 0:
            home_prob = home_prob / prob_sum if not pd.isna(home_prob) else np.nan
            draw_prob = draw_prob / prob_sum if not pd.isna(draw_prob) else np.nan
            away_prob = away_prob / prob_sum if not pd.isna(away_prob) else np.nan

        kickoff_dt = pd.to_datetime(match['kickoff_dt'], utc=True)
        days_since_home_last = np.nan
        if hh['last_dt'] is not None:
            days_since_home_last = float((kickoff_dt - hh['last_dt']).total_seconds() / 86400.0)
        days_since_away_last = np.nan
        if aa['last_dt'] is not None:
            days_since_away_last = float((kickoff_dt - aa['last_dt']).total_seconds() / 86400.0)

        league_tot_avg = mean_or_default(ll['tot'], 2.6)
        expected_goals_proxy = float(np.nanmean([
            mean_or_default(hh['gf'], 1.4), mean_or_default(aa['ga'], 1.2),
            mean_or_default(aa['gf'], 1.2), mean_or_default(hh['ga'], 1.4),
            mean_or_default(hh['home_gf'], mean_or_default(hh['gf'], 1.4)),
            mean_or_default(aa['away_ga'], mean_or_default(aa['ga'], 1.2)),
            mean_or_default(aa['away_gf'], mean_or_default(aa['gf'], 1.2)),
            mean_or_default(hh['home_ga'], mean_or_default(hh['ga'], 1.4)),
        ]))
        row = {
            'ou_line': float(match['ou_line']) if pd.notna(match.get('ou_line')) else np.nan,
            'over_odds': float(match['over_odds']) if pd.notna(match.get('over_odds')) else np.nan,
            'under_odds': float(match['under_odds']) if pd.notna(match.get('under_odds')) else np.nan,
            'fair_over_open': fair_over,
            'ou_margin_open': (over_prob + under_prob - 1.0) if not pd.isna(over_prob) and not pd.isna(under_prob) else np.nan,
            'ou_quote_count': float(match['ou_quote_count']) if pd.notna(match.get('ou_quote_count')) else np.nan,
            'home_odds': home_odds,
            'draw_odds': draw_odds,
            'away_odds': away_odds,
            'home_prob': home_prob,
            'draw_prob': draw_prob,
            'away_prob': away_prob,
            'home_away_prob_gap': (home_prob - away_prob) if not pd.isna(home_prob) and not pd.isna(away_prob) else np.nan,
            'home_gf_avg': mean_or_default(hh['gf'], 1.4),
            'home_ga_avg': mean_or_default(hh['ga'], 1.4),
            'away_gf_avg': mean_or_default(aa['gf'], 1.2),
            'away_ga_avg': mean_or_default(aa['ga'], 1.2),
            'home_tot_avg': mean_or_default(hh['tot'], league_tot_avg),
            'away_tot_avg': mean_or_default(aa['tot'], league_tot_avg),
            'home_over_rate': mean_or_default(hh['over'], 0.5),
            'away_over_rate': mean_or_default(aa['over'], 0.5),
            'home_gf_last5': mean_or_default(hh['gf'][-5:], mean_or_default(hh['gf'], 1.4)),
            'home_ga_last5': mean_or_default(hh['ga'][-5:], mean_or_default(hh['ga'], 1.4)),
            'away_gf_last5': mean_or_default(aa['gf'][-5:], mean_or_default(aa['gf'], 1.2)),
            'away_ga_last5': mean_or_default(aa['ga'][-5:], mean_or_default(aa['ga'], 1.2)),
            'home_over_last5': mean_or_default(hh['over'][-5:], mean_or_default(hh['over'], 0.5)),
            'away_over_last5': mean_or_default(aa['over'][-5:], mean_or_default(aa['over'], 0.5)),
            'home_home_gf_avg': mean_or_default(hh['home_gf'], mean_or_default(hh['gf'], 1.4)),
            'home_home_ga_avg': mean_or_default(hh['home_ga'], mean_or_default(hh['ga'], 1.4)),
            'away_away_gf_avg': mean_or_default(aa['away_gf'], mean_or_default(aa['gf'], 1.2)),
            'away_away_ga_avg': mean_or_default(aa['away_ga'], mean_or_default(aa['ga'], 1.2)),
            'home_home_over_rate': mean_or_default(hh['home_over'], mean_or_default(hh['over'], 0.5)),
            'away_away_over_rate': mean_or_default(aa['away_over'], mean_or_default(aa['over'], 0.5)),
            'expected_goals_proxy': expected_goals_proxy,
            'combined_over_rate': float(np.nanmean([
                mean_or_default(hh['over'], 0.5), mean_or_default(aa['over'], 0.5),
                mean_or_default(hh['over'][-5:], mean_or_default(hh['over'], 0.5)),
                mean_or_default(aa['over'][-5:], mean_or_default(aa['over'], 0.5)),
            ])),
            'league_tot_avg': league_tot_avg,
            'league_over_rate': mean_or_default(ll['over'], 0.5),
            'league_match_count': float(len(ll['tot'])),
            'home_match_count': float(len(hh['tot'])),
            'away_match_count': float(len(aa['tot'])),
            'days_since_home_last': days_since_home_last,
            'days_since_away_last': days_since_away_last,
            'line_minus_expected_goals': (float(match['ou_line']) - expected_goals_proxy) if pd.notna(match.get('ou_line')) else np.nan,
            'line_minus_league_avg': (float(match['ou_line']) - league_tot_avg) if pd.notna(match.get('ou_line')) else np.nan,
            'fair_over_minus_even': (fair_over - 0.5) if not pd.isna(fair_over) else np.nan,
        }
        features.append(row)

        if pd.notna(match.get('FTHG')) and pd.notna(match.get('FTAG')):
            home_goals = float(match['FTHG'])
            away_goals = float(match['FTAG'])
            total_goals = home_goals + away_goals
            ou_line = float(match['ou_line']) if pd.notna(match.get('ou_line')) else 2.5
            is_over_line = settle_binary_full_only(int(total_goals), float(ou_line))
            if is_over_line is None:
                is_over_line = 1.0 if total_goals > ou_line else 0.0

            hh['gf'].append(home_goals); hh['ga'].append(away_goals); hh['tot'].append(total_goals); hh['over'].append(float(is_over_line))
            hh['home_gf'].append(home_goals); hh['home_ga'].append(away_goals); hh['home_over'].append(float(is_over_line)); hh['last_dt'] = kickoff_dt
            aa['gf'].append(away_goals); aa['ga'].append(home_goals); aa['tot'].append(total_goals); aa['over'].append(float(is_over_line))
            aa['away_gf'].append(away_goals); aa['away_ga'].append(home_goals); aa['away_over'].append(float(is_over_line)); aa['last_dt'] = kickoff_dt
            ll['tot'].append(total_goals); ll['over'].append(float(is_over_line))

    return pd.DataFrame(features, columns=FEATURE_COLUMNS)


def build_models() -> Dict[str, Pipeline]:
    return {
        'LR': Pipeline([
            ('imp', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler()),
            ('clf', LogisticRegression(max_iter=2000, class_weight='balanced', C=0.5)),
        ]),
        'RF': Pipeline([
            ('imp', SimpleImputer(strategy='median')),
            ('clf', RandomForestClassifier(
                n_estimators=500,
                max_depth=7,
                min_samples_leaf=6,
                random_state=42,
                class_weight='balanced_subsample',
            )),
        ]),
        'ET': Pipeline([
            ('imp', SimpleImputer(strategy='median')),
            ('clf', ExtraTreesClassifier(
                n_estimators=600,
                max_depth=8,
                min_samples_leaf=5,
                random_state=42,
                class_weight='balanced',
            )),
        ]),
        'HGB': Pipeline([
            ('imp', SimpleImputer(strategy='median')),
            ('clf', HistGradientBoostingClassifier(
                max_depth=6,
                learning_rate=0.04,
                max_iter=400,
                random_state=42,
            )),
        ]),
    }


def fit_model_from_master(
    master_df: pd.DataFrame,
    thresholds: List[float],
    min_bets: int,
    min_train_rows: int,
    artifact_dir: Path,
    train_dates: List[str],
) -> TrainArtifacts:
    master_df = master_df.sort_values(['kickoff_dt', 'fixture_id']).reset_index(drop=True)
    features = build_feature_frame(master_df)

    target_mask = master_df['ou_line'].notna() & master_df['FTHG'].notna() & master_df['FTAG'].notna()
    target_df = master_df.loc[target_mask].copy().reset_index(drop=True)
    target_features = features.loc[target_mask].reset_index(drop=True)
    target_df['total_goals'] = target_df['FTHG'].astype(float) + target_df['FTAG'].astype(float)
    target_df['label'] = [settle_binary_full_only(int(tg), float(line)) for tg, line in zip(target_df['total_goals'], target_df['ou_line'])]

    dropped = int(pd.isna(target_df['label']).sum())
    usable_mask = target_df['label'].notna()
    target_df = target_df.loc[usable_mask].reset_index(drop=True)
    target_features = target_features.loc[usable_mask].reset_index(drop=True)
    target_df['label'] = target_df['label'].astype(int)

    unique_dates = sorted(target_df['Date'].unique().tolist())
    if len(unique_dates) < 4:
        raise SystemExit(f'Need at least 4 unique dates for time split, got {len(unique_dates)}')
    split_day_idx = max(1, int(len(unique_dates) * 0.8))
    split_day_idx = min(split_day_idx, len(unique_dates) - 1)
    train_day_set = set(unique_dates[:split_day_idx])
    test_day_set = set(unique_dates[split_day_idx:])

    train_mask = target_df['Date'].isin(train_day_set)
    test_mask = target_df['Date'].isin(test_day_set)
    if int(train_mask.sum()) < min_train_rows:
        raise SystemExit(f'Not enough train rows after time split: {int(train_mask.sum())} < {min_train_rows}')
    if int(test_mask.sum()) < 100:
        raise SystemExit(f'Not enough test rows after time split: {int(test_mask.sum())}')

    X_train = target_features.loc[train_mask]
    y_train = target_df.loc[train_mask, 'label']
    X_test = target_features.loc[test_mask]
    y_test = target_df.loc[test_mask, 'label']

    best = None
    best_model = None
    for model_name, model in build_models().items():
        model.fit(X_train, y_train)
        probs = model.predict_proba(X_test)[:, 1]
        preds = (probs >= 0.5).astype(int)
        overall_acc = float(accuracy_score(y_test, preds))
        for threshold in thresholds:
            mask = (probs >= threshold) | (probs <= 1 - threshold)
            bet_count = int(mask.sum())
            if bet_count < min_bets:
                continue
            bet_acc = float(accuracy_score(y_test[mask], preds[mask]))
            rec = {
                'model': model_name,
                'threshold': round(float(threshold), 2),
                'bet_count': bet_count,
                'bet_acc': bet_acc,
                'overall_acc': overall_acc,
                'rows_after_filter': int(len(target_df)),
                'train_rows': int(len(X_train)),
                'test_rows': int(len(X_test)),
                'dropped_push_or_half_rows': dropped,
                'label_policy': 'binary_full_win_only_drop_push_and_half',
                'train_dates': sorted(train_day_set),
                'test_dates': sorted(test_day_set),
                'odds_window_dates': train_dates,
            }
            if best is None or bet_acc > best['bet_acc'] or (bet_acc == best['bet_acc'] and bet_count > best['bet_count']):
                best = rec
                best_model = model
    if best is None or best_model is None:
        raise SystemExit('No candidate satisfied min_bets')

    artifact_dir.mkdir(parents=True, exist_ok=True)
    model_path = artifact_dir / 'goal_number_api_dynamic_pipeline.pkl'
    summary_path = artifact_dir / 'goal_number_api_dynamic_summary.json'
    config_path = artifact_dir / 'goal_number_api_dynamic_config.json'

    config = {
        **best,
        'task': 'goal-number-api-dynamic',
        'target': 'dynamic_ou_line',
        'features': FEATURE_COLUMNS,
        'artifact_dir': str(artifact_dir),
    }
    joblib.dump(best_model, model_path)
    summary_path.write_text(json.dumps(best, ensure_ascii=False, indent=2), encoding='utf-8')
    config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding='utf-8')

    details_path = artifact_dir / 'training_targets_used.csv'
    pd.concat([target_df.reset_index(drop=True), target_features.reset_index(drop=True)], axis=1).to_csv(details_path, index=False, encoding='utf-8-sig')
    return TrainArtifacts(best_model, best, config, model_path, summary_path, config_path)


def train_dynamic_model(args: argparse.Namespace, client: APIFootballClient) -> TrainArtifacts:
    allowed_codes = load_serving_codes()
    end_date = args.end_date or default_train_end_date()
    dates = detect_recent_odds_dates(client, end_date=end_date, max_lookback_days=args.max_lookback_days)
    if not dates:
        raise SystemExit('No recent odds dates detected from API-Football')
    target_df = collect_completed_target_rows(client, dates, allowed_codes=allowed_codes)
    if target_df.empty:
        raise SystemExit('No completed target rows extracted from recent odds window')
    history_df = collect_history_rows(client, target_df[['league_id', 'season']].drop_duplicates().itertuples(index=False, name=None), allowed_codes=allowed_codes)
    master_df = merge_history_with_targets(history_df, target_df)
    master_path = args.artifact_dir / 'training_master_dataset.csv'
    args.artifact_dir.mkdir(parents=True, exist_ok=True)
    master_df.to_csv(master_path, index=False, encoding='utf-8-sig')
    thresholds = threshold_grid(args.threshold_start, args.threshold_stop, args.threshold_step)
    artifacts = fit_model_from_master(
        master_df=master_df,
        thresholds=thresholds,
        min_bets=args.min_bets,
        min_train_rows=args.min_train_rows,
        artifact_dir=args.artifact_dir,
        train_dates=dates,
    )
    print(json.dumps(artifacts.summary, ensure_ascii=False, indent=2))
    return artifacts


def collect_upcoming_market_rows(client: APIFootballClient, now_bj: datetime, hours: int, allowed_codes: set[str]) -> pd.DataFrame:
    end = now_bj + timedelta(hours=hours)
    date_list: List[str] = []
    cur = now_bj.date()
    while cur <= end.date():
        date_list.append(cur.isoformat())
        cur += timedelta(days=1)

    fixtures: List[dict] = []
    for date_str in date_list:
        fixtures.extend(client.fixtures_by_date(date_str))
    fixture_map = {int(x['fixture']['id']): x for x in fixtures}

    odds_map: Dict[int, dict] = {}
    for date_str in date_list:
        for item in client.odds_by_date(date_str):
            odds_map[int(item['fixture']['id'])] = item

    rows: List[dict] = []
    for fixture_id, fx in fixture_map.items():
        kickoff = fixture_kickoff_dt(fx)
        if kickoff is None:
            continue
        kickoff_bj = kickoff.astimezone(BEIJING_TZ)
        if not (now_bj <= kickoff_bj <= end):
            continue
        odds_item = odds_map.get(fixture_id)
        if not odds_item:
            continue
        chosen_total = choose_main_total(odds_item.get('bookmakers', []))
        if not chosen_total:
            continue
        chosen_1x2 = choose_main_1x2(odds_item.get('bookmakers', []))
        league = fx.get('league', {})
        league_code = LEAGUE_CODE_MAP.get((league.get('country'), league.get('name')), f"api_{league.get('id')}")
        if league_code not in allowed_codes:
            continue
        rows.append({
            'fixture_id': fixture_id,
            'kickoff_dt': kickoff.isoformat(),
            'Date': kickoff_bj.strftime('%Y-%m-%d'),
            'country': league.get('country'),
            'league_name': league.get('name'),
            'league_id': int(league.get('id')),
            'season': int(league.get('season')),
            'league_code': league_code,
            'HomeTeam': normalize_team(fx['teams']['home']['name']),
            'AwayTeam': normalize_team(fx['teams']['away']['name']),
            'FTHG': np.nan,
            'FTAG': np.nan,
            **chosen_total,
            **chosen_1x2,
        })
    df = pd.DataFrame(rows)
    if df.empty:
        return df
    df['kickoff_dt'] = pd.to_datetime(df['kickoff_dt'], utc=True)
    return df.sort_values(['kickoff_dt', 'fixture_id']).reset_index(drop=True)


def recommend_level(conf: float, threshold: float) -> tuple[str, int]:
    if conf >= max(0.78, threshold + 0.08):
        return '★★★ 强烈推荐', 3
    if conf >= max(0.70, threshold):
        return '★★ 推荐', 2
    if conf >= max(0.60, threshold - 0.05):
        return '★ 谨慎推荐', 1
    return '✗ 不建议', 0


def predict_dynamic_model(args: argparse.Namespace, client: APIFootballClient) -> pd.DataFrame:
    allowed_codes = load_serving_codes()
    now_bj = parse_now_beijing(args.now)
    artifact_dir = args.artifact_dir
    model_path = artifact_dir / 'goal_number_api_dynamic_pipeline.pkl'
    config_path = artifact_dir / 'goal_number_api_dynamic_config.json'
    if not model_path.exists() or not config_path.exists():
        raise SystemExit(f'Missing trained artifact in {artifact_dir}')
    model: Pipeline = joblib.load(model_path)
    config = json.loads(config_path.read_text(encoding='utf-8'))

    future_df = collect_upcoming_market_rows(client, now_bj=now_bj, hours=args.hours, allowed_codes=allowed_codes)
    if future_df.empty:
        print('[]')
        return future_df

    history_df = collect_history_rows(client, future_df[['league_id', 'season']].drop_duplicates().itertuples(index=False, name=None), allowed_codes=allowed_codes)
    master_df = pd.concat([history_df, future_df], ignore_index=True, sort=False).sort_values(['kickoff_dt', 'fixture_id']).reset_index(drop=True)
    features = build_feature_frame(master_df)
    future_mask = master_df['FTHG'].isna() & master_df['ou_line'].notna()
    future_rows = master_df.loc[future_mask].reset_index(drop=True)
    future_features = features.loc[future_mask].reset_index(drop=True)

    probs = model.predict_proba(future_features[FEATURE_COLUMNS])[:, 1]
    out_rows: List[dict] = []
    threshold = float(config['threshold'])
    summary = json.loads((artifact_dir / 'goal_number_api_dynamic_summary.json').read_text(encoding='utf-8'))
    team_name_map = ensure_team_name_map(pd.concat([future_rows['HomeTeam'], future_rows['AwayTeam']], ignore_index=True).dropna().astype(str).tolist())
    for idx, row in future_rows.iterrows():
        over_prob = float(probs[idx])
        under_prob = float(1.0 - probs[idx])
        conf = max(over_prob, under_prob)
        pred = '大球' if over_prob >= under_prob else '小球'
        rec_label, rec_level = recommend_level(conf, threshold)
        code = row['league_code']
        league_label = LEAGUE_ZH_MAP.get(code, row['league_name'])
        out_rows.append({
            'fixture_id': int(row['fixture_id']),
            '联赛代码': code,
            '联赛': league_label,
            '联赛原名': row['league_name'],
            '国家': row['country'],
            '开赛时间': pd.to_datetime(row['kickoff_dt'], utc=True).astimezone(BEIJING_TZ).strftime('%m-%d %H:%M'),
            '主队': team_name_map.get(str(row['HomeTeam']), str(row['HomeTeam'])),
            '客队': team_name_map.get(str(row['AwayTeam']), str(row['AwayTeam'])),
            '主队原名': row['HomeTeam'],
            '客队原名': row['AwayTeam'],
            '大小球盘口': round(float(row['ou_line']), 2),
            '大球赔率': round(float(row['over_odds']), 3),
            '小球赔率': round(float(row['under_odds']), 3),
            '主胜欧赔': round(float(row['home_odds']), 3) if pd.notna(row['home_odds']) else np.nan,
            '平局欧赔': round(float(row['draw_odds']), 3) if pd.notna(row['draw_odds']) else np.nan,
            '客胜欧赔': round(float(row['away_odds']), 3) if pd.notna(row['away_odds']) else np.nan,
            '大球概率': round(over_prob * 100, 1),
            '小球概率': round(under_prob * 100, 1),
            '预测方向': pred,
            '置信度': round(conf * 100, 1),
            '是否达阈值': '是' if conf >= threshold else '否',
            '推荐等级': rec_label,
            '推荐级别': rec_level,
            '模型类型': config['model'],
            '模型阈值': round(threshold * 100, 1),
            '模型回测命中率': round(float(summary['bet_acc']) * 100, 1),
            '回测样本数': int(summary['bet_count']),
        })
    out_df = pd.DataFrame(out_rows).sort_values(['推荐级别', '置信度', '开赛时间'], ascending=[False, False, True]).reset_index(drop=True)

    args.predictions_dir.mkdir(parents=True, exist_ok=True)
    stem = f'goal_number_api_dynamic_{args.hours}h'
    csv_path = args.predictions_dir / f'{stem}.csv'
    xlsx_path = args.predictions_dir / f'{stem}.xlsx'
    out_df.to_csv(csv_path, index=False, encoding='utf-8-sig')
    out_df.to_excel(xlsx_path, index=False)

    manifest = {
        'generated_at': now_bj.strftime('%Y-%m-%d %H:%M:%S'),
        'hours': int(args.hours),
        'rows': int(len(out_df)),
        'csv': str(csv_path),
        'xlsx': str(xlsx_path),
    }
    (args.artifact_dir / 'goal_number_api_dynamic_prediction_manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding='utf-8')
    print(json.dumps(manifest, ensure_ascii=False, indent=2))
    return out_df


def build_client(args: argparse.Namespace) -> APIFootballClient:
    base_url = os.environ['API_FOOTBALL_BASE_URL'].rstrip('/')
    api_key = os.environ['API_FOOTBALL_KEY']
    return APIFootballClient(
        base_url=base_url,
        api_key=api_key,
        cache_dir=args.cache_dir,
        force_refresh=args.force_refresh,
        sleep_seconds=args.api_sleep_seconds,
    )


def main() -> None:
    args = parse_args()
    client = build_client(args)
    if args.command == 'train':
        train_dynamic_model(args, client)
        return
    if args.command == 'predict':
        predict_dynamic_model(args, client)
        return
    if args.command == 'refresh':
        train_dynamic_model(args, client)
        predict_dynamic_model(args, client)
        return
    raise SystemExit(f'Unsupported command: {args.command}')


if __name__ == '__main__':
    main()
