#!/usr/bin/env python3
"""Minimal OddsPortal bootstrap for J2 / K2 historical odds probing.

Goals:
1) discover usable results/archive paths for a league;
2) verify that rendered results pages expose match rows and pagination;
3) verify that match detail pages expose 1X2 / Asian Handicap / Over/Under fields;
4) save a compact JSON report that can be reused for later full scraping.

This intentionally prefers browser rendering (Playwright) because the archive/ajax
responses observed behind the pages are not directly consumable as plain JSON in a
simple requests-only flow.
"""

from __future__ import annotations

import argparse
import html
import json
import os
import re
import sys
import time
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Sequence

import requests

try:
    from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
    from playwright.sync_api import sync_playwright
except Exception:  # pragma: no cover - runtime fallback
    sync_playwright = None
    PlaywrightTimeoutError = Exception


USER_AGENT = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
)
BASE = "https://www.oddsportal.com"
TIMEOUT_MS = 90000

LEAGUES = {
    "j2": {
        "country": "japan",
        "slug": "j2-league",
        "label": "J2 League",
        "fallback_url": f"{BASE}/football/japan/j2-league/results/",
    },
    "k2": {
        "country": "south-korea",
        "slug": "k-league-2",
        "label": "K League 2",
        "fallback_url": f"{BASE}/football/south-korea/k-league-2/results/",
    },
    "r1": {
        "country": "russia",
        "slug": "premier-league",
        "label": "Russia Premier League",
        "fallback_url": f"{BASE}/football/russia/premier-league/results/",
    },
}


@dataclass
class Discovery:
    league: str
    label: str
    fallback_url: str
    odds_request_url: Optional[str]
    archive_seasons: List[Dict[str, str]]


class BootstrapError(RuntimeError):
    pass


def require_playwright() -> None:
    if sync_playwright is None:
        raise BootstrapError(
            "playwright python package is unavailable. "
            "Install it first (e.g. pip install playwright && playwright install chromium)."
        )


def norm_text(s: str) -> str:
    return re.sub(r"\s+", " ", s or "").strip()


def split_nonempty_lines(s: str) -> List[str]:
    return [line.strip() for line in (s or "").splitlines() if line.strip()]


def ensure_parent(path: str) -> None:
    parent = os.path.dirname(path)
    if parent:
        os.makedirs(parent, exist_ok=True)


def http_get(url: str) -> str:
    resp = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=60)
    resp.raise_for_status()
    return resp.text


def extract_odds_request_url(page_html: str) -> Optional[str]:
    match = re.search(r':odds-request="([^"]+)"', page_html)
    if match:
        raw = html.unescape(match.group(1))
        url_match = re.search(r'"url":"([^\"]+)"', raw)
        if url_match:
            return url_match.group(1).replace("\\/", "/")
    return None


def discover_routes(league: str) -> Discovery:
    meta = LEAGUES[league]
    text = http_get(meta["fallback_url"])

    option_pat = re.compile(
        rf'<option value="(https://www\.oddsportal\.com/football/{re.escape(meta["country"])}[^\"]+/results/)"[^>]*>\s*(\d{{4}})\s*</option>',
        flags=re.IGNORECASE,
    )
    seasons: List[Dict[str, str]] = []
    seen = set()
    for url, season in option_pat.findall(text):
        if url in seen:
            continue
        seen.add(url)
        seasons.append({"season": season, "url": url})

    odds_request = extract_odds_request_url(text)

    return Discovery(
        league=league,
        label=meta["label"],
        fallback_url=meta["fallback_url"],
        odds_request_url=odds_request,
        archive_seasons=seasons,
    )


def resolve_results_url(discovery: Discovery, season: Optional[str]) -> str:
    if not season:
        return discovery.fallback_url
    for row in discovery.archive_seasons:
        if row["season"] == str(season):
            return row["url"]
    raise BootstrapError(
        f"season {season} not found for {discovery.league}; available seasons: "
        f"{', '.join(r['season'] for r in discovery.archive_seasons)}"
    )


def parse_result_row_text(text: str, href: str) -> Dict[str, Any]:
    parts = split_nonempty_lines(text)
    row: Dict[str, Any] = {
        "href": href,
        "raw_parts": parts,
        "raw_text": norm_text(text),
    }
    if len(parts) >= 6 and re.match(r"^\d{1,2}:\d{2}$", parts[0]) and parts[3] == "–":
        row.update(
            {
                "kickoff_time": parts[0],
                "home_team": parts[1],
                "home_score": parts[2],
                "away_score": parts[4],
                "away_team": parts[5],
                "status_tokens": parts[6:],
            }
        )
    return row


def parse_bookmaker_1x2_row(text: str) -> Dict[str, Any]:
    parts = [p for p in split_nonempty_lines(text) if p != "CLAIM BONUS"]
    row: Dict[str, Any] = {"raw_parts": parts, "raw_text": norm_text(text)}
    if len(parts) >= 5:
        row.update(
            {
                "bookmaker": parts[0],
                "home_odds": parts[1],
                "draw_odds": parts[2],
                "away_odds": parts[3],
                "payout": parts[4],
            }
        )
    return row


def parse_game_participants(text: str) -> Dict[str, Any]:
    parts = split_nonempty_lines(text)
    row: Dict[str, Any] = {"raw_parts": parts, "raw_text": norm_text(text)}
    if len(parts) >= 5 and parts[2] == "–":
        row.update(
            {
                "home_team": parts[0],
                "home_score": parts[1],
                "away_score": parts[3],
                "away_team": parts[4],
            }
        )
    return row


def parse_collapsed_line_row(text: str, market: str) -> Dict[str, Any]:
    parts = split_nonempty_lines(text)
    row: Dict[str, Any] = {
        "market": market,
        "raw_parts": parts,
        "raw_text": norm_text(text),
    }
    if len(parts) >= 5:
        row.update(
            {
                "line": parts[0],
                "bookmaker_count": parts[1],
                "price_1": parts[2],
                "price_2": parts[3],
                "payout": parts[4],
            }
        )
        if market == "over_under":
            row["over_odds"] = parts[2]
            row["under_odds"] = parts[3]
        elif market == "asian_handicap":
            row["home_odds"] = parts[2]
            row["away_odds"] = parts[3]
    return row


def parse_expanded_line_row(text: str, market: str) -> Dict[str, Any]:
    parts = [p for p in split_nonempty_lines(text) if p != "CLAIM BONUS"]
    row: Dict[str, Any] = {
        "market": market,
        "raw_parts": parts,
        "raw_text": norm_text(text),
    }
    if len(parts) >= 5:
        row.update(
            {
                "bookmaker": parts[0],
                "line": parts[1],
                "price_1": parts[2],
                "price_2": parts[3],
                "payout": parts[4],
            }
        )
        if market == "over_under":
            row["over_odds"] = parts[2]
            row["under_odds"] = parts[3]
        elif market == "asian_handicap":
            row["home_odds"] = parts[2]
            row["away_odds"] = parts[3]
    return row


class OddsPortalBootstrapper:
    def __init__(self, headless: bool = True):
        require_playwright()
        self._pw_cm = sync_playwright()
        self._pw = self._pw_cm.start()
        self.browser = self._pw.chromium.launch(headless=headless)
        self.page = self.browser.new_page(user_agent=USER_AGENT)
        self.observed_archive_ajax_urls: List[str] = []
        self.page.on("request", self._on_request)

    def _on_request(self, req) -> None:
        url = req.url
        if "ajax-sport-country-tournament-archive_" in url and url not in self.observed_archive_ajax_urls:
            self.observed_archive_ajax_urls.append(url)

    def close(self) -> None:
        try:
            self.browser.close()
        finally:
            self._pw.stop()

    def goto(self, url: str) -> None:
        self.page.goto(url, wait_until="networkidle", timeout=TIMEOUT_MS)

    def scrape_results_page(self, url: str, page_number: int = 1, limit: int = 5) -> Dict[str, Any]:
        self.goto(url)
        pagination_numbers = []
        pagers = self.page.locator('a.pagination-link[data-number]')
        for i in range(pagers.count()):
            txt = norm_text(pagers.nth(i).inner_text())
            if txt.isdigit():
                pagination_numbers.append(int(txt))

        if page_number > 1:
            pager = self.page.locator(f'a.pagination-link[data-number="{page_number}"]')
            if pager.count() == 0:
                raise BootstrapError(f"results page {page_number} is not available; visible pages={pagination_numbers}")
            with self.page.expect_response(
                lambda r: "ajax-sport-country-tournament-archive_" in r.url and f"/page/{page_number}/" in r.url,
                timeout=TIMEOUT_MS,
            ):
                pager.first.click()
            self.page.wait_for_timeout(1500)

        rows = self.page.locator('a.next-m\\:flex')
        all_visible_match_urls: List[str] = []
        parsed_rows = []
        for i in range(rows.count()):
            row = rows.nth(i)
            href = self._abs(row.get_attribute("href"))
            if href:
                all_visible_match_urls.append(href)
            if i < limit:
                parsed_rows.append(parse_result_row_text(row.inner_text(), href))

        return {
            "results_url": url,
            "requested_page": page_number,
            "visible_pagination_numbers": pagination_numbers,
            "visible_row_count": rows.count(),
            "all_visible_match_urls": all_visible_match_urls,
            "observed_archive_ajax_urls": list(self.observed_archive_ajax_urls),
            "sample_rows": parsed_rows,
        }

    def scrape_match_markets(self, match_url: str, collapsed_limit: int = 6) -> Dict[str, Any]:
        self.goto(match_url)

        participants = parse_game_participants(self.page.locator('[data-testid="game-participants"]').inner_text())
        kickoff = norm_text(self.page.locator('[data-testid="game-time-item"]').inner_text())

        one_x_two_header = split_nonempty_lines(
            self.page.locator('[data-testid="bookmaker-table-header-line"]').first.inner_text()
        )
        one_x_two_rows = self.page.locator('[data-testid="over-under-expanded-row"]')
        one_x_two = [
            parse_bookmaker_1x2_row(one_x_two_rows.nth(i).inner_text())
            for i in range(one_x_two_rows.count())
        ]

        asian_handicap = self._scrape_line_market("Asian Handicap", market_key="asian_handicap", collapsed_limit=collapsed_limit)
        over_under = self._scrape_line_market("Over/Under", market_key="over_under", collapsed_limit=collapsed_limit)

        return {
            "match_url": match_url,
            "participants_raw": participants,
            "kickoff": kickoff,
            "one_x_two": {
                "header": one_x_two_header,
                "bookmaker_rows": one_x_two,
            },
            "asian_handicap": asian_handicap,
            "over_under": over_under,
        }

    def _scrape_line_market(self, tab_text: str, market_key: str, collapsed_limit: int = 6) -> Dict[str, Any]:
        self.page.locator('[data-testid="bet-types-nav"] li', has_text=tab_text).click()
        self.page.wait_for_load_state("networkidle", timeout=TIMEOUT_MS)
        self.page.wait_for_timeout(1200)

        header = split_nonempty_lines(self.page.locator('[data-testid="bookmaker-table-header-line"]').first.inner_text())
        rows = self.page.locator('[data-testid="over-under-collapsed-row"]')
        collapsed = [
            parse_collapsed_line_row(rows.nth(i).inner_text(), market_key)
            for i in range(min(collapsed_limit, rows.count()))
        ]

        expanded_rows: List[Dict[str, Any]] = []
        if rows.count() > 0:
            rows.first.click()
            self.page.wait_for_timeout(1000)
            exp = self.page.locator('[data-testid="over-under-expanded-row"]')
            expanded_rows = [
                parse_expanded_line_row(exp.nth(i).inner_text(), market_key)
                for i in range(exp.count())
            ]

        all_headers = [
            split_nonempty_lines(self.page.locator('[data-testid="bookmaker-table-header-line"]').nth(i).inner_text())
            for i in range(self.page.locator('[data-testid="bookmaker-table-header-line"]').count())
        ]

        return {
            "header": header,
            "collapsed_rows": collapsed,
            "headers_after_expand": all_headers,
            "expanded_rows_for_first_line": expanded_rows,
        }

    @staticmethod
    def _abs(href: Optional[str]) -> Optional[str]:
        if not href:
            return href
        if href.startswith("http://") or href.startswith("https://"):
            return href
        return BASE + href


def choose_sample_match(results_probe: Dict[str, Any], league: str, season: Optional[str]) -> Optional[str]:
    slug = LEAGUES[league]["slug"]
    season = str(season) if season else None

    all_urls = results_probe.get("all_visible_match_urls", []) or []
    for href in all_urls:
        if slug in href and (not season or season in href):
            return href

    rows = results_probe.get("sample_rows", [])
    for row in rows:
        href = row.get("href") or ""
        if slug in href and (not season or season in href):
            return href

    for href in all_urls:
        if href:
            return href
    for row in rows:
        href = row.get("href")
        if href:
            return href
    return None


def run_probe(league: str, season: Optional[str], results_page: int, limit: int, headless: bool) -> Dict[str, Any]:
    discovery = discover_routes(league)
    results_url = resolve_results_url(discovery, season)
    target_results_html_odds_request_url = extract_odds_request_url(http_get(results_url))

    bootstrapper = OddsPortalBootstrapper(headless=headless)
    try:
        results_probe = bootstrapper.scrape_results_page(results_url, page_number=results_page, limit=limit)
        sample_match = choose_sample_match(results_probe, league=league, season=season)
        market_probe = bootstrapper.scrape_match_markets(sample_match) if sample_match else None
    finally:
        bootstrapper.close()

    return {
        "generated_at_epoch": int(time.time()),
        "league": league,
        "season": str(season) if season else None,
        "discovery": {
            "league": discovery.league,
            "label": discovery.label,
            "fallback_url": discovery.fallback_url,
            "odds_request_url": discovery.odds_request_url,
            "archive_seasons": discovery.archive_seasons,
        },
        "target_results_page_odds_request_url": target_results_html_odds_request_url,
        "results_probe": results_probe,
        "sample_match_url": sample_match,
        "market_probe": market_probe,
        "notes": [
            "Archive/result rows and market tables were scraped from the rendered DOM via Playwright.",
            "Observed archive ajax endpoints exist, but direct HTTP responses are not plain JSON in a simple requests flow.",
            "Visible bookmaker coverage is geo-dependent; in this environment the rendered page mainly exposed US-facing bookmakers.",
        ],
    }


def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Minimal OddsPortal bootstrap for J2 / K2")
    sub = parser.add_subparsers(dest="command", required=True)

    probe = sub.add_parser("probe", help="discover paths and scrape small samples")
    probe.add_argument("--league", choices=sorted(LEAGUES.keys()), required=True)
    probe.add_argument("--season", help="archive season year, e.g. 2024")
    probe.add_argument("--results-page", type=int, default=1)
    probe.add_argument("--limit", type=int, default=5, help="sample row count from results page")
    probe.add_argument("--headed", action="store_true", help="run browser in headed mode")
    probe.add_argument("--output", help="optional JSON output path")

    discover = sub.add_parser("discover", help="only inspect archive paths from raw HTML")
    discover.add_argument("--league", choices=sorted(LEAGUES.keys()), required=True)
    discover.add_argument("--output", help="optional JSON output path")

    return parser.parse_args(argv)


def main(argv: Optional[Sequence[str]] = None) -> int:
    args = parse_args(argv)

    if args.command == "discover":
        discovery = discover_routes(args.league)
        payload: Dict[str, Any] = {
            "league": discovery.league,
            "label": discovery.label,
            "fallback_url": discovery.fallback_url,
            "odds_request_url": discovery.odds_request_url,
            "archive_seasons": discovery.archive_seasons,
        }
    elif args.command == "probe":
        payload = run_probe(
            league=args.league,
            season=args.season,
            results_page=args.results_page,
            limit=args.limit,
            headless=not args.headed,
        )
    else:  # pragma: no cover
        raise BootstrapError(f"unsupported command: {args.command}")

    output = json.dumps(payload, ensure_ascii=False, indent=2)
    if getattr(args, "output", None):
        ensure_parent(args.output)
        with open(args.output, "w", encoding="utf-8") as f:
            f.write(output + "\n")
    print(output)
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except (BootstrapError, requests.RequestException, PlaywrightTimeoutError) as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        raise SystemExit(1)
