#!/usr/bin/env python3
"""Scrape Ogham few meanings from dedicated sites.

Sources:
  - oghamlore.com (upright / reversed)
  - druidogham.wordpress.com (well / ill dignified)
  - globalspiritualstudies.com (chart)

Ogham has no Lenormand-style pair tables; combinations in a spread
are synthesized later from two fews' meanings.
"""
from __future__ import annotations

import html as htmlmod
import json
import re
import ssl
import time
import urllib.error
import urllib.request
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "oracle_enricher" / "out" / "ogham_scrape" / "html"
CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "ogham_corpus.json"
CTX = ssl.create_default_context()
UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
SLEEP = 0.15

# canon_id, name_en, tree_ru, druidogham path
FEDA = [
    ("beith", "Beith", "берёза", "the-fews-air/beith"),
    ("luis", "Luis", "рябина", "the-fews-air/luis"),
    ("fern", "Fearn", "ольха", "the-fews-air/fearn"),
    ("sail", "Sail", "ива", "the-fews-air/saille"),
    ("nion", "Nion", "ясень", "the-fews-air/nuin"),
    ("uath", "Huath", "боярышник", "the-fews-fire/huath"),
    ("duir", "Duir", "дуб", "the-fews-fire/duir"),
    ("tinne", "Tinne", "остролист", "the-fews-fire/tinne"),
    ("coll", "Coll", "орешник", "the-fews-fire/coll"),
    ("ceirt", "Quert", "яблоня", "the-fews-fire/quiert"),
    ("muin", "Muin", "лоза", "the-fews-water/muin"),
    ("gort", "Gort", "плющ", "the-fews-water/gort"),
    ("ngetal", "nGetal", "тростник", "the-fews-water/ngetal"),
    ("straif", "Straif", "тёрн", "the-fews-water/straif"),
    ("ruis", "Ruis", "бузина", "the-fews-water/ruis"),
    ("ailm", "Ailm", "сосна", "the-fews-earth/ailm"),
    ("onn", "Onn", "дрок", "the-fews-earth/onn"),
    ("ur", "Ur", "вереск", "the-fews-earth/ur"),
    ("eadha", "Eadha", "осина", "the-fews-earth/eadha"),
    ("idho", "Idho", "тис", "the-fews-earth/ioho"),
]


def slugify(url: str) -> str:
    s = re.sub(r"^https?://", "", url)
    return re.sub(r"[^a-zA-Z0-9._-]+", "_", s)[:180]


def fetch(url: str) -> tuple[int, str]:
    OUT.mkdir(parents=True, exist_ok=True)
    cache = OUT / (slugify(url) + ".html")
    if cache.exists() and cache.stat().st_size > 400:
        return 200, cache.read_text(encoding="utf-8", errors="replace")
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en"})
    try:
        with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
            raw = resp.read()
            enc = resp.headers.get_content_charset() or "utf-8"
            text = raw.decode(enc, errors="replace")
            status = resp.status
    except urllib.error.HTTPError as e:
        text = e.read().decode("utf-8", errors="replace") if e.fp else ""
        cache.write_text(text, encoding="utf-8")
        return e.code, text
    except Exception as e:
        return 0, str(e)
    cache.write_text(text, encoding="utf-8")
    time.sleep(SLEEP)
    return status, text


def strip_tags(html: str) -> str:
    html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
    html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
    html = re.sub(r"(?i)<br\s*/?>", "\n", html)
    html = re.sub(r"(?i)</p>", "\n\n", html)
    html = re.sub(r"(?i)</h[1-6]>", "\n", html)
    html = re.sub(r"<[^>]+>", " ", html)
    html = htmlmod.unescape(html).replace("\xa0", " ")
    html = re.sub(r"[ \t]+", " ", html)
    html = re.sub(r"\n{3,}", "\n\n", html)
    return html.strip()


def parse_oghamlore(html: str) -> dict[str, str]:
    """Map English few name -> block of upright/reversed text."""
    text = strip_tags(html)
    out = {}
    names = [en for _i, en, _t, _p in FEDA]
    # split around "Beith (B) · Birch tree" style headers
    parts = re.split(r"\n(?=(?:Beith|Luis|Fearn|Sail|Nion|Huath|Dair|Duir|Tinne|Coll|Quert|Muin|Gort|nGéadal|nGetal|Straif|Ruis|Ailm|Onn|Úr|Ur|Edad|Eadha|Idad|Idho)\b)", text)
    for part in parts:
        head = part.strip().split("\n", 1)[0][:40]
        for en in names:
            if head.lower().startswith(en.lower()) or f"{en} (" in head:
                out[en] = part.strip()[:4000]
                break
        # Dair vs Duir
        if head.lower().startswith("dair"):
            out["Duir"] = part.strip()[:4000]
    return out


def wp_content(html: str) -> str:
    m = re.search(r'(?is)<div[^>]+class="[^"]*entry-content[^"]*"[^>]*>(.*)', html)
    body = m.group(1) if m else html
    for marker in ("sharedaddy", "jp-relatedposts", "comments"):
        i = body.find(marker)
        if i > 400:
            body = body[:i]
            break
    return strip_tags(body)[:8000]


def main() -> None:
    store = {
        "meta": {
            "system": "ogham",
            "note": "Пары огама в источниках не таблицируются; сочетания — соседство fews в раскладе, сводятся из значений.",
        },
        "cards": {},
        "errors": [],
    }
    for cid, en, tree, _p in FEDA:
        store["cards"][cid] = {
            "id": cid,
            "name_en": en,
            "tree_ru": tree,
            "meanings": [],
        }

    print("== oghamlore ==")
    st, html = fetch("https://oghamlore.com/learn/ogham-divination/")
    lore = parse_oghamlore(html) if st == 200 else {}
    print("  parsed", list(lore.keys()))
    for cid, en, tree, _p in FEDA:
        if en in lore:
            store["cards"][cid]["meanings"].append(
                {"source": "oghamlore", "url": "https://oghamlore.com/learn/ogham-divination/", "text": lore[en]}
            )

    print("== druidogham ==")
    for cid, en, tree, path in FEDA:
        url = f"https://druidogham.wordpress.com/{path}/"
        st, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            print("  FAIL", cid, st)
            continue
        text = wp_content(html)
        store["cards"][cid]["meanings"].append({"source": "druidogham", "url": url, "text": text})
        print(f"  {cid:8} chars={len(text)}")

    print("== globalspiritualstudies chart ==")
    st, html = fetch("https://globalspiritualstudies.com/free-ogham-alphabet-chart/")
    if st == 200:
        text = strip_tags(html)
        store["meta"]["chart"] = text[:12000]
        print("  chart chars", len(text))

    nmean = sum(1 for c in store["cards"].values() if c["meanings"])
    CORPUS.parent.mkdir(parents=True, exist_ok=True)
    CORPUS.write_text(json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print("wrote", CORPUS, "fews_with_meanings", nmean, "errors", len(store["errors"]))


if __name__ == "__main__":
    main()