#!/usr/bin/env python3
"""Scrape Major Arcana meanings + major-major pairs from magya, karusel, ezoplaneta.

Writes oracle_enricher/data/sources/tarot_majors_corpus.json
HTML cache: oracle_enricher/out/tarot_scrape/html/
"""
from __future__ import annotations

import html as htmlmod
import json
import re
import ssl
import time
import urllib.error
import urllib.request
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
OUT_DIR = ROOT / "oracle_enricher" / "out" / "tarot_scrape"
HTML_DIR = OUT_DIR / "html"
CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "tarot_majors_corpus.json"

CTX = ssl.create_default_context()
UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
SLEEP = 0.12

# n, id, name, magya slug, karusel slug, aliases
CARDS = [
    (0, "fool", "Шут", "durak", "durak", ["шут", "дурак", "безумец"]),
    (1, "magician", "Маг", "mag", "mag", ["маг"]),
    (2, "high_priestess", "Верховная Жрица", "jrica", "verxovnaya-zhrica", ["жрица", "верховная жрица", "жрица"]),
    (3, "empress", "Императрица", "imperatrica", "imperatrica", ["императрица"]),
    (4, "emperor", "Император", "imperator", "imperator", ["император"]),
    (5, "hierophant", "Иерофант", "jrec", "ierofant", ["иерофант", "жрец", "папа"]),
    (6, "lovers", "Влюбленные", "vlublennie", "vlyublennye", ["влюбленные", "влюблённые"]),
    (7, "chariot", "Колесница", "kolesnica", "kolesnica", ["колесница"]),
    (8, "strength", "Сила", "sila", "sila", ["сила"]),
    (9, "hermit", "Отшельник", "otshelnik", "otshelnik", ["отшельник"]),
    (10, "wheel", "Колесо Фортуны", "koleso_fortuni", "koleso-fortuny", ["колесо фортуны", "колесо"]),
    (11, "justice", "Справедливость", "spravedlivost", "spravedlivost", ["справедливость", "правосудие"]),
    (12, "hanged_man", "Повешенный", "poveshenniy", "poveshennyj", ["повешенный"]),
    (13, "death", "Смерть", "smert", "smert", ["смерть"]),
    (14, "temperance", "Умеренность", "umerennost", "umerennost", ["умеренность"]),
    (15, "devil", "Дьявол", "dyavol", "dyavol", ["дьявол"]),
    (16, "tower", "Башня", "bashnya", "bashnya", ["башня"]),
    (17, "star", "Звезда", "zvezda", "zvezda", ["звезда"]),
    (18, "moon", "Луна", "luna", "luna", ["луна"]),
    (19, "sun", "Солнце", "solnce", "solnce", ["солнце"]),
    (20, "judgement", "Суд", "sud", "sud", ["суд"]),
    (21, "world", "Мир", "mir", "mir", ["мир"]),
]
BY_N = {rec[0]: rec for rec in CARDS}
MAGYA = {n: slug for n, _i, _name, slug, _k, _a in CARDS}
ALIAS = {}
for n, _i, name, _m, _k, aliases in CARDS:
    ALIAS[name.lower()] = n
    for a in aliases:
        ALIAS[a] = n


def slugify_url(url: str) -> str:
    s = re.sub(r"^https?://", "", url)
    return re.sub(r"[^a-zA-Z0-9._-]+", "_", s)[:180]


def fetch(url: str) -> tuple[int, str, str]:
    HTML_DIR.mkdir(parents=True, exist_ok=True)
    cache = HTML_DIR / (slugify_url(url) + ".html")
    if cache.exists() and cache.stat().st_size > 400:
        return 200, url, cache.read_text(encoding="utf-8", errors="replace")
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "ru"})
    try:
        with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
            raw = resp.read()
            enc = resp.headers.get_content_charset() or "utf-8"
            text = raw.decode(enc, errors="replace")
            status = resp.status
            final = str(resp.url)
    except urllib.error.HTTPError as e:
        text = e.read().decode("utf-8", errors="replace") if e.fp else ""
        cache.write_text(text, encoding="utf-8")
        return e.code, url, text
    except Exception as e:
        return 0, url, f"ERROR {type(e).__name__}: {e}"
    cache.write_text(text, encoding="utf-8")
    time.sleep(SLEEP)
    return status, final, text


def strip_tags(html: str) -> str:
    html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
    html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
    html = re.sub(r"(?i)<br\s*/?>", "\n", html)
    html = re.sub(r"(?i)</p>", "\n\n", html)
    html = re.sub(r"(?i)</h[1-6]>", "\n", html)
    html = re.sub(r"(?i)</div>", "\n", html)
    html = re.sub(r"<[^>]+>", " ", html)
    html = htmlmod.unescape(html).replace("\xa0", " ")
    html = re.sub(r"[ \t]+", " ", html)
    html = re.sub(r"\n{3,}", "\n\n", html)
    return html.strip()


def magya_main(html: str) -> str:
    m = re.search(r"(?is)<h1[^>]*>.*?</h1>(.*)", html)
    body = m.group(1) if m else html
    cut = re.search(r"(?is)(Добавить комментарий|Судьба и случай рекомендуют|Рейтинг:)", body)
    if cut:
        body = body[: cut.start()]
    text = strip_tags(body)
    text = re.sub(r"\.natal-ad-container[\s\S]{0,400}?\}", "", text)
    text = re.sub(r"@media[\s\S]{0,200}?\}", "", text)
    return re.sub(r"\n{3,}", "\n\n", text).strip()


def karusel_content(html: str) -> str:
    m = re.search(r'(?is)<div class="content clearfix">(.*)', html)
    body = m.group(1) if m else html
    for marker in ('id="sidebar', "sharedaddy", "yarpp"):
        i = body.find(marker)
        if i > 300:
            body = body[:i]
            break
    text = strip_tags(body)
    text = re.split(r"(?i)Смотрите также|Рекомендуем по этой теме", text)[0]
    return text.strip()


def empty_store() -> dict:
    cards = {}
    for n, cid, name, _m, _k, aliases in CARDS:
        cards[str(n)] = {
            "id": cid,
            "number": n,
            "name": name,
            "aliases": aliases,
            "meanings": [],
        }
    return {"meta": {"system": "tarot_majors", "cards": 22}, "cards": cards, "pairs": {}, "errors": []}


def add_meaning(store, n, source, url, text):
    text = text.strip()
    if len(text) < 40:
        return
    store["cards"][str(n)]["meanings"].append({"source": source, "url": url, "text": text})


def add_pair(store, a, b, source, url, text):
    if a == b or a < 0 or b < 0:
        return
    text = re.sub(r"\s+", " ", text).strip(" -—")
    if len(text) < 20:
        return
    key = f"{min(a,b):02d}-{max(a,b):02d}"
    bucket = store["pairs"].setdefault(key, {"a": min(a, b), "b": max(a, b), "readings": []})
    for r in bucket["readings"]:
        if r["source"] == source and r["text"] == text:
            return
    bucket["readings"].append({"source": source, "url": url, "left": a, "right": b, "text": text})


def scrape() -> dict:
    store = empty_store()
    HTML_DIR.mkdir(parents=True, exist_ok=True)

    print("== magya meanings ==")
    for n, _i, name, slug, _k, _a in CARDS:
        url = f"https://magya-online.ru/znachenie_kart_taro_raydera_weyta/{slug}"
        st, final, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            print(f"  FAIL mean {slug} {st}")
            continue
        text = magya_main(html)
        add_meaning(store, n, "magya", final, text)
        print(f"  mean {n:02d} {name} chars={len(text)}")

    print("== karusel meanings ==")
    for n, _i, name, _m, kslug, _a in CARDS:
        url = f"https://karusel-magii.ru/znachenie-kart-taro-pri-gadanii-tolkovanie-karty-taro-{kslug}/"
        st, final, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            print(f"  FAIL karusel {kslug} {st}")
            continue
        text = karusel_content(html)
        add_meaning(store, n, "karusel", final, text)
        print(f"  karusel {n:02d} {name} chars={len(text)}")

    print("== ezoplaneta love majors t=109 ==")
    st, final, html = fetch("https://ezoplaneta.ru/viewtopic.php?t=109")
    if st == 200:
        # split by ALL-CAPS names
        text = strip_tags(html)
        parts = re.split(r"\n(?=(?:ШУТ|МАГ|ВЕРХОВНАЯ ЖРИЦА|ИМПЕРАТРИЦА|ИМПЕРАТОР|ИЕРОФАНТ|ЖРЕЦ|ВЛЮБЛЕННЫЕ|КОЛЕСНИЦА|СИЛА|ОТШЕЛЬНИК|КОЛЕСО ФОРТУНЫ|СПРАВЕДЛИВОСТЬ|ПОВЕШЕННЫЙ|СМЕРТЬ|УМЕРЕННОСТЬ|ДЬЯВОЛ|БАШНЯ|ЗВЕЗДА|ЛУНА|СОЛНЦЕ|СУД|МИР)\b)", text)
        n_ezo = 0
        for part in parts:
            first = part.strip().split("\n", 1)[0][:40].lower()
            num = ALIAS.get(first.strip(" ."))
            if num is None:
                for alias, an in ALIAS.items():
                    if first.startswith(alias):
                        num = an
                        break
            if num is None:
                continue
            add_meaning(store, num, "ezoplaneta", final, part.strip())
            n_ezo += 1
        print(f"  ezo cards {n_ezo}")
    else:
        store["errors"].append({"url": "ezo t=109", "status": st})

    print("== magya major-major pair essays ==")
    ok = 0
    total = 0
    for a, _ia, _na, sa, _ka, _aa in CARDS:
        for b, _ib, _nb, sb, _kb, _ab in CARDS:
            if a == b:
                continue
            total += 1
            url = f"https://magya-online.ru/sochetanie_kart_taro/{sa}_i_{sb}"
            st, final, html = fetch(url)
            if st != 200:
                store["errors"].append({"url": url, "status": st})
                continue
            text = magya_main(html)
            if "выберите сочетание" in text.lower()[:400]:
                continue
            if len(text) < 80:
                continue
            add_pair(store, a, b, "magya_pair", final, text)
            ok += 1
        print(f"  from {a:02d} pairs ok so far {ok}")
    print(f"  pair essays kept={ok}/{total}")

    # short combos from magya hubs
    print("== magya hub short combos ==")
    nshort = 0
    for n, _i, name, slug, _k, _a in CARDS:
        url = f"https://magya-online.ru/sochetanie_kart_taro/{slug}"
        st, final, html = fetch(url)
        if st != 200:
            continue
        text = magya_main(html)
        for m in re.finditer(
            r"(?m)^([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-\s]{2,40}?)\s*[—–\-]\s*(.+)$",
            text,
        ):
            right = ALIAS.get(m.group(1).strip().lower())
            if right is None:
                right = ALIAS.get(m.group(1).strip().lower().split()[0])
            if right is None or right == n:
                continue
            add_pair(store, n, right, "magya_meaning", final, m.group(2))
            nshort += 1
    print(f"  short combos {nshort}")
    return store


def main() -> None:
    store = scrape()
    store["meta"]["pair_count"] = len(store["pairs"])
    store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
    CORPUS.parent.mkdir(parents=True, exist_ok=True)
    CORPUS.write_text(json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print("wrote", CORPUS)
    print("cards with meanings", sum(1 for c in store["cards"].values() if c["meanings"]))
    print("pairs", store["meta"]["pair_count"], "readings", store["meta"]["reading_count"])
    print("errors", len(store["errors"]))


if __name__ == "__main__":
    main()