#!/usr/bin/env python3
"""
pro_v5 数据下载 - 快速版
修复: 不使用 page 参数 (API-Football page=1 对某些联赛返回0)
STEP 1: 下载所有 fixture（无 page 参数）
STEP 2: 获取 B365 让球盘赔率（高并发）
"""
import os, sys, time, json
from pathlib import Path
from datetime import datetime, timezone, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed
import numpy as np
import pandas as pd
import requests

__import__('warnings').filterwarnings('ignore')
np.random.seed(42)

REPO = Path('/root/.openclaw/workspace-flj_bot1/football-predictor-repro')
ODDS_DIR = REPO / 'artifacts' / 'odds_api_history'
ODDS_DIR.mkdir(parents=True, exist_ok=True)

API_KEY = os.environ.get('API_FOOTBALL_KEY', '57b64e857ebeffdc156f0c41d43c7894')
BASE = 'https://v3.football.api-sports.io'
HEADERS = {'x-apisports-key': API_KEY}
TZ = timezone(timedelta(hours=8))

LEAGUE_IDS = {
    'e0':39,'e1':40,'e2':41,'e2n':42,'e3':183,
    'd1':78,'d2':79,'f1':61,'f2':62,
    'i1':135,'i2':136,'sp1':140,'sp2':141,
    'tr1':203,'p1':94,'b1':145,'g1':197,
    'n1':88,'n2':89,'sc1':180,'cz1':237,
    'a1':279,'r1':164,'br1':71,'m1':253,'j1':98,
}
SEASONS = [2023, 2024, 2025]
BET365_ID = 6
MAX_WORKERS = 20

def get_fixtures(league_code, league_id, season):
    cache = ODDS_DIR / f'{league_code}_S{season}_fixtures.csv'
    if cache.exists() and cache.stat().st_size > 200:
        try:
            ex = pd.read_csv(cache, usecols=['FixtureID'])
            if len(ex) > 100:
                print(f"  {league_code} S{season}: 已缓存 {len(ex)} fixtures ✓", flush=True)
                return cache
        except: pass
    
    print(f"  {league_code} S{season}: 下载...", end=' ', flush=True)
    rows = []
    page = 1
    while True:
        try:
            params = {'league': league_id, 'season': season}
            if page > 1:
                params['page'] = page
            r = requests.get(f'{BASE}/fixtures', params=params, headers=HEADERS, timeout=30)
            if r.status_code != 200:
                print(f"HTTP {r.status_code}", flush=True)
                break
            d = r.json()
            fixtures = d.get('response', [])
            if not fixtures:
                break
            for fix in fixtures:
                fi = fix['fixture']
                teams = fix['teams']
                goals = fix.get('goals', {})
                rows.append({
                    'Date': datetime.fromtimestamp(fi['timestamp'], tz=TZ).strftime('%Y-%m-%d'),
                    'LeagueCode': league_code, 'LeagueID': league_id, 'Season': season,
                    'HomeTeam': teams['home']['name'], 'AwayTeam': teams['away']['name'],
                    'HomeTeamID': teams['home'].get('id'), 'AwayTeamID': teams['away'].get('id'),
                    'FTHG': goals.get('home'), 'FTAG': goals.get('away'),
                    'Status': fi.get('status', {}).get('short', ''),
                    'FixtureID': fi['id'],
                    'HasOdds': False, 'B365AHH': np.nan, 'B365AHA': np.nan, 'AHh': np.nan,
                })
            paging = d.get('paging', {})
            total = int(paging.get('total', 1))
            if page >= total:
                break
            page += 1
            time.sleep(0.05)
        except Exception as e:
            print(f"错误: {e}", flush=True)
            break
    
    if rows:
        df = pd.DataFrame(rows)
        df.to_csv(cache, index=False, encoding='utf-8')
        print(f"{len(df)} fixtures ✓", flush=True)
        return cache
    print("0 fixtures ⚠️", flush=True)
    return None

def fetch_odds(fid):
    time.sleep(0.04)
    try:
        r = requests.get(f'{BASE}/odds',
            params={'fixture': fid, 'bookmaker': BET365_ID},
            headers=HEADERS, timeout=15)
        if r.status_code != 200:
            return None
        for item in r.json().get('response', []):
            for bm in item.get('bookmakers', []):
                if str(bm.get('id')) != str(BET365_ID):
                    continue
                for bet in bm.get('bets', []):
                    name = bet.get('name', '')
                    if name not in ('Asian Handicap', 'Handicap', 'Half Time Asian Handicap'):
                        continue
                    vals = bet.get('values', [])
                    if len(vals) < 2:
                        continue
                    handicap = None
                    home_odd = away_odd = None
                    for v in vals:
                        hcap = v.get('handicap', '')
                        if hcap:
                            try:
                                handicap = float(hcap)
                            except:
                                pass
                        if v.get('type') == 'Home' or 'home' in v.get('name', '').lower():
                            try: home_odd = float(v.get('odd'))
                            except: pass
                        elif v.get('type') == 'Away' or 'away' in v.get('name', '').lower():
                            try: away_odd = float(v.get('odd'))
                            except: pass
                    if home_odd and away_odd and handicap is not None:
                        return {'fixture_id': fid, 'B365AHH': home_odd, 'B365AHA': away_odd, 'AHh': handicap}
    except:
        pass
    return None

def process_league(league_code, league_id, season):
    cache = get_fixtures(league_code, league_id, season)
    if cache is None or not cache.exists():
        return 0
    try:
        df = pd.read_csv(cache, low_memory=False)
    except:
        return 0
    need_odds = df[df['B365AHH'].isna() | (df['B365AHH'] < 1.1)]['FixtureID'].tolist()
    if not need_odds:
        valid = int((df['B365AHH'].notna() & (df['B365AHH'] > 1.0)).sum())
        print(f"  {league_code} S{season}: 已有 {valid} 条赔率 ✓", flush=True)
        return valid
    print(f"  {league_code} S{season}: 获取 {len(need_odds)} 个 B365 赔率...", flush=True)
    results = {}
    done = 0
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
        futures = {ex.submit(fetch_odds, fid): fid for fid in need_odds}
        for future in as_completed(futures):
            res = future.result()
            if res:
                results[res['fixture_id']] = res
            done += 1
            if done % 100 == 0:
                print(f"    {done}/{len(need_odds)} ({len(results)} 条有效)", flush=True)
    updated = 0
    for fid, odds in results.items():
        mask = df['FixtureID'] == fid
        if mask.any():
            for col in ['B365AHH', 'B365AHA', 'AHh']:
                if odds.get(col) is not None:
                    df.loc[mask, col] = odds[col]
            df.loc[mask, 'HasOdds'] = True
            updated += 1
    df.to_csv(cache, index=False, encoding='utf-8')
    valid = int((df['B365AHH'].notna() & (df['B365AHH'] > 1.0)).sum())
    print(f"  → {updated} 条更新，有效赔率 {valid}/{len(df)}", flush=True)
    return valid

def merge_league(code):
    files = sorted(ODDS_DIR.glob(f'{code}_S*_fixtures.csv'))
    if not files:
        return 0
    dfs = []
    for f in files:
        try:
            dfs.append(pd.read_csv(f, low_memory=False))
        except: pass
    if not dfs:
        return 0
    combined = pd.concat(dfs, ignore_index=True).drop_duplicates(subset=['FixtureID'])
    combined['Date'] = pd.to_datetime(combined['Date'], errors='coerce')
    combined = combined.sort_values('Date').reset_index(drop=True)
    out = ODDS_DIR / f'{code}_full_odds.csv'
    combined.to_csv(out, index=False, encoding='utf-8')
    valid = int((combined['B365AHH'].notna() & (combined['B365AHH'] > 1.0)).sum())
    print(f"  {code}: {len(combined)} 条 → {valid} 有效赔率", flush=True)
    return valid

print("=" * 60)
print("pro_v5 数据下载（快速版）")
print("=" * 60)
print(f"API: {API_KEY[:10]}...")
print(f"联赛: {len(LEAGUE_IDS)} × {len(SEASONS)} 赛季 = {len(LEAGUE_IDS)*len(SEASONS)} 任务")
print(f"并发: {MAX_WORKERS}")
print("=" * 60)

total_valid = 0
tasks = [(code, lid, s) for code, lid in LEAGUE_IDS.items() for s in SEASONS]
for i, (code, lid, season) in enumerate(tasks):
    print(f"[{i+1}/{len(tasks)}]", end=' ')
    total_valid += process_league(code, lid, season)

print("\n" + "=" * 60)
print("合并数据...")
print("=" * 60)
summary = []
for code in sorted(LEAGUE_IDS.keys()):
    v = merge_league(code)
    summary.append((code, v))

print("\n" + "=" * 60)
print("最终结果")
print("=" * 60)
print(f"{'联赛':<8} {'有效B365':>10} {'状态'}")
for code, v in summary:
    print(f"{code:<8} {v:>10} {'✅' if v > 0 else '❌'}")
print(f"\n总计: {sum(v for _, v in summary)} 条有效 B365 数据")
