feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets

scripts/export_lenormand_atlas.py

@@ -0,0 +1,349 @@
+#!/usr/bin/env python3
+"""Build UI-ready Lenormand pairs asset from the scraped corpus.
+
+Writes:
+  assets/data/lenormand_pairs.json
+  oracle_enricher/data/lenormand_pairs.json  (same blob)
+Also supplements traditions_canon.json lore from Magya short meanings.
+"""
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "lenormand_corpus.json"
+CANON = ROOT / "oracle_enricher" / "data" / "traditions_canon.json"
+UNIFIED = ROOT / "oracle_enricher" / "out" / "lenormand_unify" / "unified.json"
+OUT_ASSET = ROOT / "assets" / "data" / "lenormand_pairs.json"
+OUT_DATA = ROOT / "oracle_enricher" / "data" / "lenormand_pairs.json"
+HTML_CACHE = ROOT / "oracle_enricher" / "out" / "lenormand_scrape" / "html"
+ASSET_CANON = ROOT / "assets" / "data" / "traditions_canon.json"
+HTML_VIEWER = ROOT / "lexicon_viewer.html"
+
+KARUSEL_MEAN_SLUG = {
+    1: "karty-lenorman-vsadnik",
+    2: "karty-lenorman-klever",
+    3: "karty-lenorman-korabl",
+    4: "karty-lenorman-dom",
+    5: "karty-lenorman-derevo",
+    6: "karty-lenorman-tuchi",
+    7: "karty-lenorman-zmeya",
+    8: "karty-lenorman-grob",
+    9: "karty-lenorman-buket",
+    10: "karty-lenorman-kosa",
+    11: "karty-lenorman-metla",
+    12: "karty-lenorman-sovy",
+    13: "karty-lenorman-rebenok",
+    14: "karty-lenorman-lisa",
+    15: "karty-lenorman-medved",
+    16: "karty-lenorman-zvezdy",
+    17: "karty-lenorman-aist",
+    18: "karty-lenorman-sobaka",
+    19: "karty-lenorman-bashnya",
+    20: "karty-lenorman-sad",
+    21: "karty-lenorman-gora",
+    22: "karty-lenorman-razvilka",
+    23: "karty-lenorman-krysy",
+    24: "karta-lenorman-serdce",
+    25: "karty-lenorman-kolco",
+    26: "karty-lenorman-kniga",
+    27: "karty-lenorman-pismo",
+    28: "karty-lenorman-muzhchina",
+    29: "karty-lenorman-zhenshhina",
+    30: "karty-lenorman-lilii",
+    31: "karty-lenorman-solnce",
+    32: "karty-lenorman-luna",
+    33: "karty-lenorman-klyuch",
+    34: "karty-lenorman-ryby",
+    35: "karty-lenorman-yakor",
+    36: "karty-lenorman-krest",
+}
+
+SOURCE_LABELS = {
+    "karusel_table": "Карусель · таблица",
+    "karusel_short": "Карусель · кратко",
+    "ezoplaneta": "Эзопланета",
+    "magya_meaning": "Magya · кратко",
+    "magya_pair": "Magya · эссе",
+    "karusel": "Карусель",
+    "magya": "Magya",
+}
+
+SOURCE_ORDER = [
+    "magya_pair",
+    "karusel_table",
+    "magya_meaning",
+    "ezoplaneta",
+    "karusel_short",
+]
+
+
+def sort_readings(readings: list[dict]) -> list[dict]:
+    rank = {s: i for i, s in enumerate(SOURCE_ORDER)}
+    return sorted(readings, key=lambda r: (rank.get(r.get("source"), 99), r.get("left", 0)))
+
+
+def strip_tags(html: str) -> str:
+    html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
+    html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
+    html = re.sub(r"(?i)<br\s*/?>", "\n", html)
+    html = re.sub(r"(?i)</p>", "\n\n", html)
+    html = re.sub(r"(?i)</h[1-6]>", "\n", html)
+    html = re.sub(r"(?i)</div>", "\n", html)
+    html = re.sub(r"<[^>]+>", " ", html)
+    html = html.replace("\xa0", " ")
+    html = re.sub(r"[ \t]+", " ", html)
+    html = re.sub(r"\n[ \t]+", "\n", html)
+    html = re.sub(r"\n{3,}", "\n\n", html)
+    return html.strip()
+
+
+def clean_magya_meaning(text: str) -> str:
+    m = re.search(r"Краткое значение:", text)
+    if m:
+        text = text[m.start() :]
+    text = re.split(r"(?i)Сочетание карты .+ с другими картами", text)[0]
+    text = re.sub(r"(?is)Гадания\s*>[\s\S]{0,200}?Всадник\s*", "", text)
+    text = re.sub(r"^\s*>\s*", "", text, flags=re.M)
+    return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def clean_ezo_meaning(text: str) -> str:
+    text = re.sub(r"(?im)^.*\.(jpg|png|jpeg).*$", "", text)
+    text = re.sub(r"(?im)^.*\d[\d\s.,]*\s*КБ.*просмотр.*$", "", text)
+    text = re.split(r"(?i)\nСочетани", text)[0]
+    text = re.sub(r"\n{3,}", "\n\n", text).strip()
+    return text
+
+
+def clean_karusel_from_cache(n: int, fallback: str) -> str:
+    slug = KARUSEL_MEAN_SLUG.get(n)
+    if not slug:
+        return fallback
+    path = HTML_CACHE / f"karusel-magii.ru_{slug}_.html"
+    if not path.exists():
+        path = HTML_CACHE / f"karusel-magii.ru_{slug}.html"
+    if not path.exists():
+        return fallback
+    html = path.read_text(encoding="utf-8", errors="replace")
+    m = re.search(r'(?is)<div class="content clearfix">(.*)', html)
+    if not m:
+        return fallback
+    body = m.group(1)
+    for marker in ('id="sidebar', 'id="footer', "sharedaddy", "yarpp"):
+        i = body.find(marker)
+        if i > 200:
+            body = body[:i]
+            break
+    text = strip_tags(body)
+    text = re.split(r"(?i)Смотрите также", text)[0]
+    text = re.split(r"(?i)Рекомендуем по этой теме", text)[0]
+    text = re.sub(r"(?im)^Карты Ленорман .*$", "", text)
+    text = re.sub(r"\n{3,}", "\n\n", text).strip()
+    return text if len(text) > 80 else fallback
+
+
+def preview_of(readings: list[dict]) -> str:
+    for src in SOURCE_ORDER:
+        for r in readings:
+            if r.get("source") == src and r.get("text"):
+                t = r["text"].replace("\n", " ").strip()
+                return t if len(t) <= 160 else t[:157].rstrip() + "…"
+    return ""
+
+
+def main() -> None:
+    corpus = json.loads(CORPUS.read_text(encoding="utf-8"))
+    canon = json.loads(CANON.read_text(encoding="utf-8"))
+    unified = {}
+    if UNIFIED.exists():
+        unified = json.loads(UNIFIED.read_text(encoding="utf-8"))
+    stations = canon["lenormand"]["stations"]
+    by_n: dict[int, dict] = {}
+    for st in stations:
+        sid = st.get("id", "")
+        if sid.startswith("card_"):
+            n = int(sid.split("_")[1])
+            by_n[n] = st
+
+    cards_out = []
+    for n in range(1, 37):
+        cc = corpus["cards"][str(n)]
+        st = by_n.get(n, {})
+        meanings = []
+        magya_short = ""
+        for m in cc.get("meanings", []):
+            src = m.get("source", "")
+            text = (m.get("text") or "").strip()
+            if src == "magya":
+                text = clean_magya_meaning(text)
+                mm = re.search(r"Краткое значение:\s*(.+)", text)
+                if mm:
+                    magya_short = mm.group(1).split("\n")[0].strip(" .")
+            elif src == "ezoplaneta":
+                text = clean_ezo_meaning(text)
+            elif src == "karusel":
+                text = clean_karusel_from_cache(n, text)
+            if len(text) < 40:
+                continue
+            meanings.append(
+                {
+                    "source": src,
+                    "label": SOURCE_LABELS.get(src, src),
+                    "text": text,
+                }
+            )
+        pairs = []
+        for other in range(1, 37):
+            if other == n:
+                continue
+            key = f"{min(n, other):02d}-{max(n, other):02d}"
+            bucket = corpus["pairs"].get(key)
+            if not bucket:
+                continue
+            readings = []
+            for r in sort_readings(bucket.get("readings", [])):
+                readings.append(
+                    {
+                        "source": r.get("source"),
+                        "label": SOURCE_LABELS.get(r.get("source", ""), r.get("source", "")),
+                        "left": r.get("left"),
+                        "right": r.get("right"),
+                        "text": (r.get("text") or "").strip(),
+                    }
+                )
+            if not readings:
+                continue
+            ost = by_n.get(other, {})
+            up = (unified.get("pairs") or {}).get(key) or {}
+            utext = (up.get("text") or "").strip()
+            pairs.append(
+                {
+                    "n": other,
+                    "id": f"card_{other}",
+                    "name": corpus["cards"][str(other)]["name"],
+                    "symbol": ost.get("symbol", ""),
+                    "preview": (utext[:157] + "…") if len(utext) > 160 else (utext or preview_of(readings)),
+                    "text": utext,
+                    "readings": [] if utext else readings,
+                }
+            )
+        ucard = (unified.get("cards") or {}).get(str(n)) or {}
+        umeaning = (ucard.get("meaning") or "").strip()
+        ucore = (ucard.get("core") or "").strip()
+        cards_out.append(
+            {
+                "n": n,
+                "id": f"card_{n}",
+                "name": cc["name"],
+                "symbol": st.get("symbol", ""),
+                "lore": ucore or magya_short or st.get("lore", ""),
+                "group": st.get("group", f"Карта {n}"),
+                "meaning": umeaning,
+                "shadow": (ucard.get("shadow") or "").strip(),
+                "meanings": [] if umeaning else meanings,
+                "pairs": pairs,
+            }
+        )
+
+    names = {c["n"]: c["name"] for c in cards_out}
+    triads = []
+    uni_triads = unified.get("triads") or []
+    if uni_triads:
+        for t in uni_triads:
+            nums = [int(x) for x in t.get("cards", [])]
+            if len(nums) != 3:
+                continue
+            triads.append(
+                {
+                    "cards": nums,
+                    "names": [names.get(x, str(x)) for x in nums],
+                    "text": (t.get("text") or "").strip(),
+                }
+            )
+    else:
+        for t in corpus.get("triads", []):
+            nums = [int(x) for x in t.get("cards", []) if str(x).isdigit()]
+            if len(nums) != 3:
+                continue
+            triads.append(
+                {
+                    "cards": nums,
+                    "names": [names.get(x, str(x)) for x in nums],
+                    "text": (t.get("text") or "").strip(),
+                }
+            )
+
+    extras = []
+    for e in corpus.get("extras", []):
+        title = (e.get("title") or "").strip()
+        text = (e.get("text") or "").strip()
+        if len(text) < 80:
+            continue
+        extras.append(
+            {
+                "source": e.get("source", ""),
+                "title": title,
+                "text": text,
+            }
+        )
+
+    blob = {
+        "system": "lenormand",
+        "source_labels": SOURCE_LABELS,
+        "card_count": 36,
+        "pair_count": sum(len(c["pairs"]) for c in cards_out) // 2,
+        "cards": cards_out,
+        "triads": triads,
+        "extras": extras,
+    }
+    text = json.dumps(blob, ensure_ascii=False, indent=2) + "\n"
+    OUT_ASSET.parent.mkdir(parents=True, exist_ok=True)
+    OUT_ASSET.write_text(text, encoding="utf-8")
+    OUT_DATA.write_text(text, encoding="utf-8")
+
+    # Supplement station lore in traditions canon (short line on the card).
+    lore_by_n = {c["n"]: c["lore"] for c in cards_out if c.get("lore")}
+    for canon_path in (CANON, ASSET_CANON):
+        data = json.loads(canon_path.read_text(encoding="utf-8"))
+        ln = data["lenormand"]
+        ln["description"] = (
+            "36 карт французского оракула мадемуазель Ленорман: значения с трёх источников "
+            "и 630 парных сочетаний (карусель, эзопланета, magya) — источники не схлопываются."
+        )
+        for st in ln["stations"]:
+            sid = st.get("id", "")
+            if not sid.startswith("card_"):
+                continue
+            n = int(sid.split("_")[1])
+            if lore_by_n.get(n):
+                st["lore"] = lore_by_n[n]
+        canon_path.write_text(
+            json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+        )
+
+    if HTML_VIEWER.exists():
+        html = HTML_VIEWER.read_text(encoding="utf-8")
+        start = html.find("    const TRADITIONS_CANON = ")
+        end = html.find("    const WORDS_DATA = ")
+        if start > 0 and end > start:
+            data = json.loads(CANON.read_text(encoding="utf-8"))
+            blob_js = (
+                "    const TRADITIONS_CANON = "
+                + json.dumps(data, ensure_ascii=False, indent=2)
+                + ";\n\n"
+            )
+            HTML_VIEWER.write_text(html[:start] + blob_js + html[end:], encoding="utf-8")
+            print("patched", HTML_VIEWER.relative_to(ROOT))
+
+    print(
+        f"cards={len(cards_out)} pairs_per_card={len(cards_out[0]['pairs'])} "
+        f"triads={len(triads)} extras={len(extras)} bytes={len(text)}"
+    )
+    print("wrote", OUT_ASSET.relative_to(ROOT))
+
+
+if __name__ == "__main__":
+    main()