#!/usr/bin/env python3
"""
从 API-Football 高效抓取历史赔率数据（多线程并发版）
策略：
1. 先批量获取所有赛季的已完成比赛（不带赔率）
2. 并发获取赔率（每场1次API调用）
3. 限速：每秒最多5个请求
"""
from __future__ import annotations
import concurrent.futures, json, math, os, re, sys, time, threading, queue
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional
import requests
import numpy as np
import pandas as pd

REPO_ROOT = Path(__file__).resolve().parents[1]
HIST_DIR = REPO_ROOT / 'historical_data'
CACHE_DIR = REPO_ROOT / 'artifacts' / 'api_fast_cache'
BEIJING_TZ = timezone(timedelta(hours=8))
CACHE_DIR.mkdir(parents=True, exist_ok=True)

# 联赛配置：(本地代码, API联赛ID, 联赛名, 已有比赛数据的年份数)
# 有历史数据但缺OU的：n2, r1, a1 → 只需补充赔率
LEAGUES = [
    ('n2',  89,  '荷兰乙级联赛',  True),
    ('r1',  235, '俄罗斯超级联赛', True),
    ('a1',  218, '奥地利超级联赛', True),
    ('s1',  207, '瑞士超级联赛', False),
    ('dk1', 119, '丹麦超级联赛', False),
    ('no1', 103, '挪威超级联赛', False),
    ('no2', 104, '挪威甲级联赛', False),
    ('sw1', 113, '瑞典超级联赛', False),
    ('sw2', 114, '瑞典甲级联赛', False),
    ('pl1', 106, '波兰超级联赛', False),
    ('hu1', 271, '匈牙利甲级联赛', False),
    ('ro1', 283, '罗马尼亚甲级联赛', False),
    ('rs1', 286, '塞尔维亚超级联赛', False),
    ('bg1', 172, '保加利亚超级联赛', False),
    ('il1', 383, '以色列超级联赛', False),
    ('sk1', 332, '斯洛伐克超级联赛', False),
    ('fi1', 244, '芬兰超级联赛', False),
    ('ir1', 357, '爱尔兰超级联赛', False),
    ('is1', 164, '冰岛超级联赛', False),
]

RATE_LIMIT = 5  # 每秒最多5个请求

def normalize_team(name: str) -> str:
    if not name: return ''
    s = re.sub(r'[^a-zA-Z0-9 ]', ' ', str(name)).strip()
    return re.sub(r'\s+', ' ', s)

def extract_total_value(raw: str):
    if not raw: return None
    m = re.match(r'^(Over|Under)\s+([0-9.]+)$', raw.strip(), re.IGNORECASE)
    if not m: return None
    return m.group(1).lower(), float(m.group(2))

def settle_binary(total_goals: float, line: float):
    frac = round(line - math.floor(line), 2)
    base = math.floor(line)
    if frac == 0.0: return 1 if total_goals > line else (0 if total_goals < line else None)
    if frac == 0.5: return 1 if total_goals > line else 0
    if frac == 0.25:
        if total_goals >= base + 1: return 1
        if total_goals <= base - 1: return 0
        return None
    if frac == 0.75:
        if total_goals >= base + 2: return 1
        if total_goals <= base: return 0
        return None
    return None

# ─── 限速请求 ──────────────────────────────────────────────────────────────

class RateLimitedClient:
    def __init__(self):
        self.base_url = os.environ.get('API_FOOTBALL_BASE_URL', '').rstrip('/')
        self.api_key = os.environ.get('API_FOOTBALL_KEY', '')
        self.lock = threading.Lock()
        self.last_call = 0.0
        self.call_count = 0
        self.session = requests.Session()
        self.session.headers['x-apisports-key'] = self.api_key

    def _rate_limit(self):
        with self.lock:
            now = time.time()
            if now - self.last_call < 1.0:
                wait = 1.0 - (now - self.last_call)
                time.sleep(wait)
            self.last_call = time.time()
            self.call_count += 1

    def get(self, endpoint: str, params: dict) -> dict:
        self._rate_limit()
        r = self.session.get(self.base_url + endpoint, params=params, timeout=30)
        if r.status_code in (403, 429):
            print(f'  ⚠️ Rate limited! waiting 60s...')
            time.sleep(60)
            self._rate_limit()
            r = self.session.get(self.base_url + endpoint, params=params, timeout=30)
        r.raise_for_status()
        return r.json()

    def get_league_seasons(self, league_id: int) -> list:
        resp = self.get('/leagues', {'id': league_id})
        seasons = []
        for item in resp.get('response', []):
            for s in item.get('seasons', []):
                if s.get('year'): seasons.append(s['year'])
        return sorted(seasons)

    def get_fixtures(self, league_id: int, season: int) -> list:
        resp = self.get('/fixtures', {'league': league_id, 'season': season})
        return resp.get('response', [])

    def get_odds(self, fixture_id: int) -> list:
        resp = self.get('/odds', {'fixture': fixture_id})
        return resp.get('response', [])


# ─── 赔率解析 ──────────────────────────────────────────────────────────────

def parse_odds(bookmakers: list) -> Optional[dict]:
    per_line = defaultdict(lambda: {'over': [], 'under': []})
    for bookmaker in bookmakers:
        for bet in bookmaker.get('bets', []):
            if bet.get('name') != 'Goals Over/Under': continue
            for item in bet.get('values', []):
                parsed = extract_total_value(item.get('value'))
                if not parsed: continue
                side, line = parsed
                try: odd = float(item.get('odd'))
                except: continue
                if odd <= 1: continue
                per_line[line][side].append(odd)

    if not per_line: return None
    results = []
    for line, sides in per_line.items():
        if 'over' not in sides or 'under' not in sides: continue
        ov = float(np.mean(sides['over']))
        un = float(np.mean(sides['under']))
        ov_p = 1.0/ov if ov > 1 else np.nan
        un_p = 1.0/un if un > 1 else np.nan
        tot = ov_p + un_p
        results.append({
            'line': float(line),
            'over_odds': ov, 'under_odds': un,
            'fair_over': ov_p/tot if tot > 0 else np.nan,
        })
    if not results: return None
    results.sort(key=lambda x: abs((x['fair_over'] if not math.isnan(x['fair_over']) else 0.5) - 0.5), reverse=True)
    best = results[0]
    return {
        'ou_line': best['line'],
        'over_odds': best['over_odds'],
        'under_odds': best['under_odds'],
        'fair_over': best['fair_over'],
    }


# ─── 单联赛处理 ──────────────────────────────────────────────────────────

def process_league(local_code: str, league_id: int, league_name: str, has_fixtures: bool) -> dict:
    print(f'  [{league_name}] 开始处理...')
    client = RateLimitedClient()
    local_path = HIST_DIR / f'{local_code}_history.csv'

    # 读取已有数据的 fixture_id + (Date,Home,Away) 用于去重
    existing_keys = {}
    existing_ids = set()
    has_existing = False
    if local_path.exists():
        try:
            df = pd.read_csv(local_path, encoding='utf-8-sig', low_memory=False)
            if 'Date' in df.columns and 'HomeTeam' in df.columns:
                has_existing = True
                for _, row in df.iterrows():
                    key = (str(row.get('Date','')), str(row.get('HomeTeam','')), str(row.get('AwayTeam','')))
                    existing_keys[key] = row.get('FTHG', np.nan)
                    existing_ids.add(row.get('fixture_id', 0))
        except: pass

    # 获取赛季列表
    seasons = client.get_league_seasons(league_id)
    if not seasons:
        print(f'  ⚠️ 无法获取赛季')
        return {'league': league_name, 'status': 'FAILED'}

    # 收集所有比赛
    all_fixtures = []
    for season in seasons:
        print(f'  赛季 {season}...', end=' ', flush=True)
        fixtures = client.get_fixtures(league_id, season)
        print(f'{len(fixtures)} 场')
        for f in fixtures:
            status = f['fixture']['status']['short']
            if status not in ('FT', 'AET', 'PEN', 'FT'):
                continue
            fid = f['fixture']['id']
            home = normalize_team(f['teams']['home']['name'])
            away = normalize_team(f['teams']['away']['name'])
            ks = f['fixture'].get('date', '')
            try:
                kt = datetime.fromisoformat(ks.replace('Z','+00:00')).replace(tzinfo=timezone.utc).astimezone(BEIJING_TZ)
                date_str = kt.strftime('%Y-%m-%d')
            except:
                date_str = ks[:10] if ks else ''
            key = (date_str, home, away)
            fthg = int(f['goals']['home'])
            ftag = int(f['goals']['away'])
            all_fixtures.append({
                'fixture_id': fid, 'Date': date_str,
                'HomeTeam': home, 'AwayTeam': away,
                'FTHG': fthg, 'FTAG': ftag,
                'FTR': 'H' if fthg > ftag else ('A' if fthg < ftag else 'D'),
                'season': season,
                'has_existing': key in existing_keys,
                'key': key,
            })

    # 对缺少OU的联赛，只需要补赔率
    need_odds = not has_existing  # 全新联赛需要赔率

    # 并发获取赔率（只对新的/缺的fixture）
    fixture_ids = [f['fixture_id'] for f in all_fixtures]
    new_fixture_ids = [fid for fid in fixture_ids if fid not in existing_ids]

    odds_map = {}
    if new_fixture_ids:
        print(f'  并发获取 {len(new_fixture_ids)} 场赔率...')
        with concurrent.futures.ThreadPoolExecutor(max_workers=RATE_LIMIT) as ex:
            futures = {ex.submit(client.get_odds, fid): fid for fid in new_fixture_ids}
            done = 0
            for fut in concurrent.futures.as_completed(futures):
                fid = futures[fut]
                try:
                    odds_list = fut.result()
                    for odds_item in odds_list:
                        bms = odds_item.get('bookmakers', [])
                        if bms:
                            ou = parse_odds(bms)
                            if ou:
                                odds_map[fid] = ou
                                break
                except Exception as e:
                    pass
                done += 1
                if done % 100 == 0:
                    print(f'    进度 {done}/{len(new_fixture_ids)}')

    # 构建新行
    new_rows = []
    for f in all_fixtures:
        if f['has_existing']: continue  # 已有数据，跳过
        fid = f['fixture_id']
        row = {
            'fixture_id': fid, 'Date': f['Date'],
            'HomeTeam': f['HomeTeam'], 'AwayTeam': f['AwayTeam'],
            'FTHG': f['FTHG'], 'FTAG': f['FTAG'],
            'FTR': f['FTR'], 'season': f['season'],
        }
        ou = odds_map.get(fid)
        if ou:
            row['B365>2.5'] = ou['over_odds']
            row['B365<2.5'] = ou['under_odds']
            row['Avg>2.5'] = ou['over_odds']
            row['Avg<2.5'] = ou['under_odds']
            row['AvgC>2.5'] = ou['over_odds']
            row['AvgC<2.5'] = ou['under_odds']
        new_rows.append(row)

    ou_count = sum(1 for r in new_rows if 'B365>2.5' in r)

    # 合并
    if new_rows:
        new_df = pd.DataFrame(new_rows)
        if has_existing:
            old_df = pd.read_csv(local_path, encoding='utf-8-sig', low_memory=False)
            combined = pd.concat([old_df, new_df], ignore_index=True)
        else:
            combined = new_df
        combined.to_csv(local_path, index=False, encoding='utf-8-sig')
        print(f'  ✅ {league_name}: 新增 {len(new_rows)} 行 (含OU: {ou_count}), 总 {len(combined)} 行')
        return {
            'league': league_name, 'status': 'OK' if ou_count else 'NO_OU',
            'new_rows': len(new_rows), 'ou_count': ou_count,
            'total': len(combined),
        }
    elif has_existing:
        print(f'  ⏭️ {league_name}: 无新数据')
        return {'league': league_name, 'status': 'SKIP', 'total': sum(1 for _ in open(local_path)) - 1}
    else:
        print(f'  ⚠️ {league_name}: 无新数据')
        return {'league': league_name, 'status': 'EMPTY'}


# ─── 主函数 ──────────────────────────────────────────────────────────────

def main():
    results = []
    for local_code, league_id, name, has_fixtures in LEAGUES:
        print(f'\n[{local_code}] {name} (ID={league_id})')
        try:
            r = process_league(local_code, league_id, name, has_fixtures)
            results.append(r)
        except Exception as e:
            print(f'  ❌ 异常: {e}')
            results.append({'league': name, 'status': 'ERROR', 'error': str(e)})
        time.sleep(2)

    print('\n========== 汇总 ==========')
    ok = [r for r in results if r.get('status') == 'OK']
    skip = [r for r in results if r.get('status') == 'SKIP']
    empty = [r for r in results if r.get('status') == 'EMPTY']
    fail = [r for r in results if r.get('status') in ('FAILED', 'ERROR')]
    print(f'✅ 成功(含OU): {len(ok)} 联赛')
    print(f'⏭️  跳过: {len(skip)} 联赛')
    print(f'⚠️  无数据: {len(empty)} 联赛')
    print(f'❌ 失败: {len(fail)} 联赛')
    for r in ok:
        print(f'  ✅ {r["league"]}: 新增{r["new_rows"]}行, 总{r["total"]}行, OU={r["ou_count"]}')
    for r in skip:
        print(f'  ⏭️ {r["league"]}: 已有{r["total"]}行')
    for r in fail:
        print(f'  ❌ {r["league"]}: {r.get("error","")}')


if __name__ == '__main__':
    main()
