#!/usr/bin/env python3
"""Scrape Lenormand meanings + pair combinations from three Russian sources.

Sources (kept separate — they disagree):
  karusel   https://karusel-magii.ru/
  ezoplaneta https://ezoplaneta.ru/viewtopic.php?t=1913
  magya     https://magya-online.ru/

Writes:
  oracle_enricher/data/sources/lenormand_corpus.json
  oracle_enricher/out/lenormand_scrape/coverage.md
  oracle_enricher/out/lenormand_scrape/html/  (raw cache)
"""
from __future__ import annotations

import html as htmlmod
import json
import re
import ssl
import time
import urllib.error
import urllib.request
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
OUT_DIR = ROOT / "oracle_enricher" / "out" / "lenormand_scrape"
HTML_DIR = OUT_DIR / "html"
CORPUS_PATH = ROOT / "oracle_enricher" / "data" / "sources" / "lenormand_corpus.json"
COV_PATH = OUT_DIR / "coverage.md"

CTX = ssl.create_default_context()
UA = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
)
SLEEP = 0.12

# number, id, canonical RU, aliases (lowercase)
CARDS = [
    (1, "rider", "Всадник", ["всадник", "гонец"]),
    (2, "clover", "Клевер", ["клевер"]),
    (3, "ship", "Корабль", ["корабль"]),
    (4, "house", "Дом", ["дом"]),
    (5, "tree", "Дерево", ["дерево"]),
    (6, "clouds", "Тучи", ["тучи", "облака", "туча", "облако"]),
    (7, "snake", "Змея", ["змея"]),
    (8, "coffin", "Гроб", ["гроб"]),
    (9, "bouquet", "Букет", ["букет"]),
    (10, "scythe", "Коса", ["коса"]),
    (11, "whip", "Метла", ["метла", "метлы", "розги"]),
    (12, "birds", "Птицы", ["птицы", "совы", "сова", "птица"]),
    (13, "child", "Ребенок", ["ребенок", "ребёнок", "дитя"]),
    (14, "fox", "Лиса", ["лиса"]),
    (15, "bear", "Медведь", ["медведь"]),
    (16, "stars", "Звезды", ["звезды", "звезда", "звёзды"]),
    (17, "stork", "Аист", ["аист", "аисты"]),
    (18, "dog", "Собака", ["собака"]),
    (19, "tower", "Башня", ["башня"]),
    (20, "garden", "Сад", ["сад", "парк"]),
    (21, "mountain", "Гора", ["гора"]),
    (22, "crossroads", "Развилка", ["развилка", "перекресток", "перекрёсток", "развилки"]),
    (23, "mice", "Крысы", ["крысы", "крыса", "мыши", "мышь"]),
    (24, "heart", "Сердце", ["сердце"]),
    (25, "ring", "Кольцо", ["кольцо"]),
    (26, "book", "Книга", ["книга"]),
    (27, "letter", "Письмо", ["письмо"]),
    (28, "man", "Мужчина", ["мужчина", "барин"]),
    (29, "woman", "Женщина", ["женщина", "барыня"]),
    (30, "lilies", "Лилии", ["лилии", "лилия"]),
    (31, "sun", "Солнце", ["солнце"]),
    (32, "moon", "Луна", ["луна"]),
    (33, "key", "Ключ", ["ключ"]),
    (34, "fish", "Рыбы", ["рыбы", "рыба"]),
    (35, "anchor", "Якорь", ["якорь"]),
    (36, "cross", "Крест", ["крест"]),
]
BY_NUM = {n: (i, name, aliases) for n, i, name, aliases in CARDS}
ALIAS_TO_NUM: dict[str, int] = {}
for n, _i, name, aliases in CARDS:
    ALIAS_TO_NUM[name.lower()] = n
    for a in aliases:
        ALIAS_TO_NUM[a] = n

MAGYA_SLUG = {
    1: "vsadnik", 2: "klever", 3: "korabl", 4: "dom", 5: "derevo",
    6: "tuchi", 7: "zmeya", 8: "grob", 9: "buket", 10: "kosa",
    11: "metla", 12: "sovi", 13: "rebenok", 14: "lisa", 15: "medved",
    16: "zvezdi", 17: "aist", 18: "sobaka", 19: "bashnya", 20: "sad",
    21: "gora", 22: "razvilka", 23: "krisi", 24: "serdce", 25: "kolco",
    26: "kniga", 27: "pismo", 28: "mujchina", 29: "jenschina", 30: "lilii",
    31: "solnce", 32: "luna", 33: "kluch", 34: "ribi", 35: "yakor", 36: "krest",
}

KARUSEL_COMBO = {
    1: "karta-vsadnik-lenorman",
    2: "karta-klever-lenorman",
    3: "karta-korabl-lenorman",
    4: "karta-dom-lenorman",
    5: "karta-derevo-lenorman",
    6: "karta-oblaka-lenorman",
    7: "karta-zmeya-lenorman",
    8: "karta-grob-lenorman",
    9: "karta-buket-lenorman",
    10: "karta-kosa-lenorman",
    11: "karta-metla-lenorman",
    12: "karta-ptizi-lenorman",
    13: "karta-rebenok-lenorman",
    14: "karta-lisa-lenorman",
    15: "karta-medved-lenorman",
    16: "karta-zvezdy-lenorman",
    17: "karta-aist-lenorman",
    18: "karta-sobaka-lenorman",
    19: "karta-bashnya-lenorman",
}

KARUSEL_MEAN = {
    1: "karty-lenorman-vsadnik",
    2: "karty-lenorman-klever",
    3: "karty-lenorman-korabl",
    4: "karty-lenorman-dom",
    5: "karty-lenorman-derevo",
    6: "karty-lenorman-tuchi",
    7: "karty-lenorman-zmeya",
    8: "karty-lenorman-grob",
    9: "karty-lenorman-buket",
    10: "karty-lenorman-kosa",
    11: "karty-lenorman-metla",
    12: "karty-lenorman-sovy",
    13: "karty-lenorman-rebenok",
    14: "karty-lenorman-lisa",
    15: "karty-lenorman-medved",
    16: "karty-lenorman-zvezdy",
    17: "karty-lenorman-aist",
    18: "karty-lenorman-sobaka",
    19: "karty-lenorman-bashnya",
    20: "karty-lenorman-sad",
    21: "karty-lenorman-gora",
    22: "karty-lenorman-razvilka",
    23: "karty-lenorman-krysy",
    24: "karta-lenorman-serdce",
    25: "karty-lenorman-kolco",
    26: "karty-lenorman-kniga",
    27: "karty-lenorman-pismo",
    28: "karty-lenorman-muzhchina",
    29: "karty-lenorman-zhenshhina",
    30: "karty-lenorman-lilii",
    31: "karty-lenorman-solnce",
    32: "karty-lenorman-luna",
    33: "karty-lenorman-klyuch",
    34: "karty-lenorman-ryby",
    35: "karty-lenorman-yakor",
    36: "karty-lenorman-krest",
}


def slugify_url(url: str) -> str:
    s = re.sub(r"^https?://", "", url)
    s = re.sub(r"[^a-zA-Z0-9._-]+", "_", s)
    return s[:180]


def fetch(url: str, force: bool = False) -> tuple[int, str, str]:
    HTML_DIR.mkdir(parents=True, exist_ok=True)
    cache = HTML_DIR / (slugify_url(url) + ".html")
    if cache.exists() and not force and cache.stat().st_size > 400:
        return 200, url, cache.read_text(encoding="utf-8", errors="replace")
    req = urllib.request.Request(
        url,
        headers={
            "User-Agent": UA,
            "Accept-Language": "ru,en;q=0.8",
            "Accept": "text/html,application/xhtml+xml",
        },
    )
    try:
        with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
            raw = resp.read()
            enc = resp.headers.get_content_charset() or "utf-8"
            text = raw.decode(enc, errors="replace")
            final = str(resp.url)
            status = resp.status
    except urllib.error.HTTPError as e:
        text = e.read().decode("utf-8", errors="replace") if e.fp else ""
        cache.write_text(text, encoding="utf-8")
        return e.code, url, text
    except Exception as e:
        return 0, url, f"ERROR {type(e).__name__}: {e}"
    cache.write_text(text, encoding="utf-8")
    time.sleep(SLEEP)
    return status, final, text


def strip_tags(html: str) -> str:
    html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
    html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
    html = re.sub(r"(?is)<noscript[^>]*>.*?</noscript>", " ", html)
    html = re.sub(r"(?i)<br\s*/?>", "\n", html)
    html = re.sub(r"(?i)</p>", "\n\n", html)
    html = re.sub(r"(?i)</div>", "\n", html)
    html = re.sub(r"(?i)</li>", "\n", html)
    html = re.sub(r"(?i)</h[1-6]>", "\n", html)
    html = re.sub(r"(?i)</tr>", "\n", html)
    html = re.sub(r"(?i)</td>", " | ", html)
    html = re.sub(r"(?i)</th>", " | ", html)
    html = re.sub(r"<[^>]+>", " ", html)
    html = htmlmod.unescape(html)
    html = html.replace("\xa0", " ").replace("&nbsp;", " ")
    html = re.sub(r"[ \t]+", " ", html)
    html = re.sub(r"\n[ \t]+", "\n", html)
    html = re.sub(r"\n{3,}", "\n\n", html)
    return html.strip()


def magya_main(html: str) -> str:
    m = re.search(r"(?is)<h1[^>]*>.*?</h1>(.*)", html)
    body = m.group(1) if m else html
    cut = re.search(
        r"(?is)(Добавить комментарий|Судьба и случай рекомендуют|Рейтинг:)",
        body,
    )
    if cut:
        body = body[: cut.start()]
    text = strip_tags(body)
    # drop leftover css
    text = re.sub(r"\.r_adapt_zag[\s\S]{0,400}?\}", "", text)
    text = re.sub(r"blockquote,[\s\S]{0,500}?inside;\s*\}", "", text)
    text = re.sub(r"li p \{[^}]+\}", "", text)
    text = re.sub(r"Гадания\s*>\s*.*?Всадник.*?\n", "", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def karusel_content(html: str) -> str:
    m = re.search(r'(?is)<div[^>]+class="[^"]*entry-content[^"]*"[^>]*>(.*)', html)
    body = m.group(1) if m else html
    cut = re.search(r'(?is)<div[^>]+class="[^"]*(sharedaddy|yarpp|comments)', body)
    if cut:
        body = body[: cut.start()]
    return strip_tags(body)


def resolve_name(token: str) -> int | None:
    t = token.strip().lower()
    t = t.replace("ё", "е")
    t = re.sub(r"[^а-яa-z0-9]+", " ", t).strip()
    if not t:
        return None
    if t in ALIAS_TO_NUM:
        return ALIAS_TO_NUM[t]
    # first word
    w = t.split()[0]
    return ALIAS_TO_NUM.get(w)


def pair_key(a: int, b: int) -> str:
    return f"{min(a, b):02d}-{max(a, b):02d}"


def add_meaning(store, num: int, source: str, url: str, kind: str, text: str):
    text = text.strip()
    if len(text) < 40:
        return
    store["cards"][str(num)]["meanings"].append(
        {"source": source, "kind": kind, "url": url, "text": text}
    )


def add_pair(store, a: int, b: int, source: str, url: str, text: str, order: str):
    if a == b:
        return
    text = re.sub(r"\s+", " ", text).strip(" -—–|")
    if len(text) < 8:
        return
    key = pair_key(a, b)
    bucket = store["pairs"].setdefault(
        key,
        {"a": min(a, b), "b": max(a, b), "readings": []},
    )
    # de-dupe identical source+text
    for r in bucket["readings"]:
        if r["source"] == source and r["text"] == text:
            return
    bucket["readings"].append(
        {
            "source": source,
            "url": url,
            "order": order,  # a_then_b | undirected
            "left": a,
            "right": b,
            "text": text,
        }
    )


def parse_karusel_combo_table(html: str, left: int, url: str, store) -> int:
    n = 0
    # rows: number | NAME | meaning
    for m in re.finditer(
        r"(?is)<tr[^>]*>\s*<td[^>]*>\s*(\d{1,2})\s*</td>\s*<td[^>]*>\s*([^<]+)</td>\s*<td[^>]*>(.*?)</td>",
        html,
    ):
        right = int(m.group(1))
        meaning = strip_tags(m.group(3))
        if right == left or right < 1 or right > 36:
            continue
        add_pair(store, left, right, "karusel_table", url, meaning, "a_then_b")
        n += 1
    if n:
        return n
    # markdown-like leftover or plain | 1 | ВСАДНИК | text |
    text = strip_tags(html)
    for m in re.finditer(
        r"(?m)^\s*(\d{1,2})\s*\|\s*([А-ЯЁа-яёA-Za-z\s]+)\s*\|\s*(.+)$",
        text,
    ):
        right = int(m.group(1))
        meaning = m.group(3).strip()
        if right == left or right < 1 or right > 36:
            continue
        add_pair(store, left, right, "karusel_table", url, meaning, "a_then_b")
        n += 1
    return n


def parse_karusel_short_pairs(html: str, url: str, store) -> int:
    text = karusel_content(html)
    n = 0
    current_left = None
    for line in text.splitlines():
        line = line.strip()
        m_head = re.match(
            r"^(?:#{0,6}\s*)?(\d{1,2})\.\s*([А-ЯЁа-яёA-Za-z]+)\s*\+",
            line,
        )
        if m_head:
            current_left = int(m_head.group(1))
            continue
        m_pair = re.match(
            r"^(\d{1,2})\s+([А-ЯЁа-яёA-Za-z\-]+)\s*[—–\-]\s*(.+)$",
            line,
        )
        if m_pair and current_left:
            right = int(m_pair.group(1))
            add_pair(
                store,
                current_left,
                right,
                "karusel_short",
                url,
                m_pair.group(3),
                "a_then_b",
            )
            n += 1
    return n


def parse_karusel_triads(html: str, url: str, store) -> int:
    text = karusel_content(html)
    n = 0
    for m in re.finditer(
        r"(\d{1,2})\s*\+\s*(\d{1,2})\s*\+\s*(\d{1,2})\s*=\s*(.+)",
        text,
    ):
        a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
        if not all(1 <= x <= 36 for x in (a, b, c)):
            continue
        store["triads"].append(
            {
                "cards": [a, b, c],
                "source": "karusel_triad",
                "url": url,
                "text": m.group(4).strip(),
            }
        )
        n += 1
    return n


def extract_divs_by_class(html: str, cls: str) -> list[str]:
    """Return inner HTML of each <div class="cls"> with nested divs balanced."""
    pat = re.compile(rf'<div\s+class="{re.escape(cls)}">', re.I)
    out: list[str] = []
    i = 0
    while True:
        m = pat.search(html, i)
        if not m:
            break
        pos = m.end()
        depth = 1
        j = pos
        captured = None
        while j < len(html) and depth:
            nxt_open = html.find("<div", j)
            nxt_close = html.find("</div>", j)
            if nxt_close < 0:
                break
            if nxt_open != -1 and nxt_open < nxt_close:
                depth += 1
                j = nxt_open + 4
            else:
                depth -= 1
                if depth == 0:
                    captured = html[pos:nxt_close]
                    i = nxt_close + 6
                    break
                j = nxt_close + 6
        if captured is None:
            break
        out.append(captured)
    return out


def parse_ezo_thread(html: str, url: str, store) -> dict:
    stats = {"meanings": 0, "pairs": 0, "extras": 0}
    posts = extract_divs_by_class(html, "content")
    extra_re = re.compile(
        r"(?i)перевёрнут|карта дня|кратк\w+ значен|комбинац|магическ|кармическ|финансов|любви|работ"
    )
    for raw in posts:
        text = strip_tags(raw)
        current_left = None
        head = re.search(
            r"(?i)(?:^|\n)\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\s]{1,24})\s*(?:\(|$)",
            text,
        )
        if not head:
            head_html = re.search(
                r'color:#008000["\']?[^>]*>\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][^<(]{1,40})',
                raw,
            )
            if head_html:
                class _H:
                    def group(self, i):
                        return head_html.group(i)
                head = _H()
        if head:
            num = int(head.group(1)) if head.group(1) else resolve_name(head.group(2))
            if isinstance(num, int) and 1 <= num <= 36:
                current_left = num
                add_meaning(store, num, "ezoplaneta", url, "card", text)
                stats["meanings"] += 1
        elif extra_re.search(text[:400]) and len(text) > 200:
            title = re.sub(r"\s+", " ", text.split("\n", 1)[0])[:90]
            store["extras"].append(
                {"source": "ezoplaneta", "url": url, "title": title, "text": text}
            )
            stats["extras"] += 1

        if not current_left:
            continue
        # Early posts: <strong>КЛЕВЕР</strong>-Скорое...
        # Later posts: <strong>Всадник</strong><br>- Неожиданная...
        for m in re.finditer(
            r"<strong[^>]*>\s*([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-\s]{2,24}?)\s*</strong>"
            r"\s*(?:<br\s*/?>|\s)*[-—–:]?\s*([^<]+)",
            raw,
        ):
            right = resolve_name(m.group(1))
            meaning = m.group(2).strip()
            if not right or right == current_left:
                continue
            if len(meaning) < 4:
                continue
            add_pair(
                store,
                current_left,
                right,
                "ezoplaneta",
                url,
                meaning,
                "a_then_b",
            )
            stats["pairs"] += 1
    return stats


def parse_magya_meaning(html: str, num: int, url: str, store) -> int:
    text = magya_main(html)
    # split off combinations section
    parts = re.split(r"(?i)Сочетание карты .+? с другими картами", text, maxsplit=1)
    main = parts[0].strip()
    add_meaning(store, num, "magya", url, "card", main)
    n = 0
    if len(parts) > 1:
        combo_block = parts[1]
        for m in re.finditer(
            r"(?m)^([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-]{2,20})\s*[—–\-]\s*(.+)$",
            combo_block,
        ):
            right = resolve_name(m.group(1))
            if not right or right == num:
                continue
            add_pair(store, num, right, "magya_meaning", url, m.group(2), "a_then_b")
            n += 1
    return n


def parse_magya_pair(html: str, url: str, store) -> bool:
    m = re.search(r"/sochetanie_kart_lenorman/([a-z]+)_i_([a-z]+)", url)
    if not m:
        return False
    slug_a, slug_b = m.group(1), m.group(2)
    inv = {v: k for k, v in MAGYA_SLUG.items()}
    a = inv.get(slug_a)
    b = inv.get(slug_b)
    if not a or not b:
        return False
    text = magya_main(html)
    text = re.sub(r"(?is)Смотрите также:[\s\S]+$", "", text).strip()
    text = re.sub(r"(?is)Гадания\s*>[\s\S]{0,250}?\n+", "", text)
    text = re.sub(
        r"(?is)^(?:Значение карт Ленорман\s*>\s*)?(?:Сочетание карт Ленорман\s*>\s*)?[А-ЯЁа-яёA-Za-z\s]+и [А-ЯЁа-яёA-Za-z\s]+\s*",
        "",
        text,
        count=1,
    )
    text = re.sub(r"\s+", " ", text).strip()
    if len(text) < 40:
        return False
    add_pair(store, a, b, "magya_pair", url, text, "a_then_b")
    return True


def empty_store() -> dict:
    cards = {}
    for n, cid, name, aliases in CARDS:
        cards[str(n)] = {
            "id": cid,
            "number": n,
            "name": name,
            "aliases": aliases,
            "meanings": [],
        }
    return {
        "meta": {
            "system": "lenormand",
            "cards": 36,
            "sources": {
                "karusel": "https://karusel-magii.ru/",
                "ezoplaneta": "https://ezoplaneta.ru/viewtopic.php?t=1913",
                "magya": "https://magya-online.ru/",
            },
            "note": "Источники расходятся: readings[] хранит все версии без схлопывания.",
        },
        "cards": cards,
        "pairs": {},
        "triads": [],
        "extras": [],
        "errors": [],
    }


def scrape(limit_pairs: int | None = None) -> dict:
    store = empty_store()
    HTML_DIR.mkdir(parents=True, exist_ok=True)

    # ---------- KARUSEL ----------
    print("== karusel meanings ==")
    for n, slug in KARUSEL_MEAN.items():
        url = f"https://karusel-magii.ru/{slug}/"
        st, final, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            print(f"  FAIL {n} {slug} {st}")
            continue
        text = karusel_content(html)
        add_meaning(store, n, "karusel", final, "card", text)
        print(f"  mean {n:02d} {BY_NUM[n][1]} chars={len(text)}")

    url = "https://karusel-magii.ru/znachenie-kart-lenorman/"
    st, final, html = fetch(url)
    if st == 200:
        add_meaning(store, 1, "karusel", final, "hub_skipped", "")  # noop
        # keep as extra hub
        store["extras"].append(
            {
                "source": "karusel",
                "url": final,
                "title": "Значение карт Ленорман (сводная)",
                "text": karusel_content(html)[:20000],
            }
        )

    print("== karusel combo tables ==")
    for n, slug in KARUSEL_COMBO.items():
        url = f"https://karusel-magii.ru/{slug}/"
        st, final, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            print(f"  FAIL combo {n} {st}")
            continue
        k = parse_karusel_combo_table(html, n, final, store)
        print(f"  table {n:02d} {BY_NUM[n][1]} pairs={k}")

    print("== karusel short pairs + triads ==")
    url = "https://karusel-magii.ru/znachenie-i-sochetanie-kart-lenorman/"
    st, final, html = fetch(url)
    if st == 200:
        k = parse_karusel_short_pairs(html, final, store)
        print(f"  short pairs {k}")
    url = "https://karusel-magii.ru/sochetanie-kart-lenorman/"
    st, final, html = fetch(url)
    if st == 200:
        t = parse_karusel_triads(html, final, store)
        print(f"  triads {t}")

    # ---------- EZOPLANETA ----------
    print("== ezoplaneta thread ==")
    for start in (0, 10, 20, 30, 40):
        url = (
            "https://ezoplaneta.ru/viewtopic.php?t=1913"
            if start == 0
            else f"https://ezoplaneta.ru/viewtopic.php?t=1913&start={start}"
        )
        st, final, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            print(f"  FAIL start={start} {st}")
            continue
        stats = parse_ezo_thread(html, final, store)
        print(f"  page start={start} {stats}")

    # ---------- MAGYA meanings ----------
    print("== magya meanings ==")
    pair_urls: list[str] = []
    for n, slug in MAGYA_SLUG.items():
        url = f"https://magya-online.ru/znachenie_kart_lenorman/{slug}"
        st, final, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            print(f"  FAIL mean {slug} {st}")
            continue
        k = parse_magya_meaning(html, n, final, store)
        print(f"  mean {n:02d} {slug} combos={k}")

        hub = f"https://magya-online.ru/sochetanie_kart_lenorman/{slug}"
        st2, final2, html2 = fetch(hub)
        if st2 == 200:
            found = re.findall(
                rf"""href=['"](/sochetanie_kart_lenorman/{slug}_i_[a-z]+)['"]""",
                html2,
            )
            for rel in found:
                pair_urls.append("https://magya-online.ru" + rel)

    # also generate directed slugs — magya uses cardA_i_cardB
    for a, sa in MAGYA_SLUG.items():
        for b, sb in MAGYA_SLUG.items():
            if a == b:
                continue
            pair_urls.append(
                f"https://magya-online.ru/sochetanie_kart_lenorman/{sa}_i_{sb}"
            )

    # unique preserve order
    seen = set()
    uniq = []
    for u in pair_urls:
        if u not in seen:
            seen.add(u)
            uniq.append(u)
    if limit_pairs:
        uniq = uniq[:limit_pairs]
    print(f"== magya pair essays {len(uniq)} ==")
    ok = 0
    for i, url in enumerate(uniq, 1):
        st, final, html = fetch(url)
        if st != 200:
            store["errors"].append({"url": url, "status": st})
            continue
        if parse_magya_pair(html, final, store):
            ok += 1
        if i % 50 == 0:
            print(f"  pairs {i}/{len(uniq)} ok={ok}")
    print(f"  pair essays ok={ok}/{len(uniq)}")

    return store


def coverage(store: dict) -> str:
    lines = [
        "# Lenormand corpus coverage",
        "",
        f"Карт: {len(store['cards'])}",
        f"Пар с хотя бы одним чтением: {len(store['pairs'])} / 630 undirected",
        f"Триад: {len(store['triads'])}",
        f"Доп. разделов: {len(store['extras'])}",
        f"Ошибок загрузки: {len(store['errors'])}",
        "",
        "## Значения по картам (число источников)",
        "",
    ]
    for n in range(1, 37):
        c = store["cards"][str(n)]
        srcs = sorted({m["source"] for m in c["meanings"] if m.get("text")})
        lines.append(f"- {n:02d} {c['name']}: {len(c['meanings'])} записей ({', '.join(srcs) or '—'})")

    by_src = defaultdict(int)
    disagree = 0
    for key, bucket in store["pairs"].items():
        srcs = {r["source"] for r in bucket["readings"]}
        for s in srcs:
            by_src[s] += 1
        if len(srcs) >= 2:
            disagree += 1
    lines += [
        "",
        "## Пары по источнику",
        "",
    ]
    for s, n in sorted(by_src.items()):
        lines.append(f"- {s}: {n}")
    lines += [
        "",
        f"Пар, где источники расходятся (2+ источника): {disagree}",
        "",
    ]
    return "\n".join(lines) + "\n"


def main() -> None:
    import argparse

    ap = argparse.ArgumentParser()
    ap.add_argument("--limit-pairs", type=int, default=None)
    ap.add_argument("--force", action="store_true")
    args = ap.parse_args()
    if args.force:
        for p in HTML_DIR.glob("*.html"):
            p.unlink()
    store = scrape(limit_pairs=args.limit_pairs)
    store["meta"]["pair_count"] = len(store["pairs"])
    store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
    CORPUS_PATH.parent.mkdir(parents=True, exist_ok=True)
    CORPUS_PATH.write_text(
        json.dumps(store, ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )
    cov = coverage(store)
    COV_PATH.write_text(cov, encoding="utf-8")
    print("\n" + cov)
    print("wrote", CORPUS_PATH)
    print("wrote", COV_PATH)


if __name__ == "__main__":
    main()