350 lines
12 KiB
Python
350 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Build UI-ready Lenormand pairs asset from the scraped corpus.
|
|
|
|
Writes:
|
|
assets/data/lenormand_pairs.json
|
|
oracle_enricher/data/lenormand_pairs.json (same blob)
|
|
Also supplements traditions_canon.json lore from Magya short meanings.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "lenormand_corpus.json"
|
|
CANON = ROOT / "oracle_enricher" / "data" / "traditions_canon.json"
|
|
UNIFIED = ROOT / "oracle_enricher" / "out" / "lenormand_unify" / "unified.json"
|
|
OUT_ASSET = ROOT / "assets" / "data" / "lenormand_pairs.json"
|
|
OUT_DATA = ROOT / "oracle_enricher" / "data" / "lenormand_pairs.json"
|
|
HTML_CACHE = ROOT / "oracle_enricher" / "out" / "lenormand_scrape" / "html"
|
|
ASSET_CANON = ROOT / "assets" / "data" / "traditions_canon.json"
|
|
HTML_VIEWER = ROOT / "lexicon_viewer.html"
|
|
|
|
KARUSEL_MEAN_SLUG = {
|
|
1: "karty-lenorman-vsadnik",
|
|
2: "karty-lenorman-klever",
|
|
3: "karty-lenorman-korabl",
|
|
4: "karty-lenorman-dom",
|
|
5: "karty-lenorman-derevo",
|
|
6: "karty-lenorman-tuchi",
|
|
7: "karty-lenorman-zmeya",
|
|
8: "karty-lenorman-grob",
|
|
9: "karty-lenorman-buket",
|
|
10: "karty-lenorman-kosa",
|
|
11: "karty-lenorman-metla",
|
|
12: "karty-lenorman-sovy",
|
|
13: "karty-lenorman-rebenok",
|
|
14: "karty-lenorman-lisa",
|
|
15: "karty-lenorman-medved",
|
|
16: "karty-lenorman-zvezdy",
|
|
17: "karty-lenorman-aist",
|
|
18: "karty-lenorman-sobaka",
|
|
19: "karty-lenorman-bashnya",
|
|
20: "karty-lenorman-sad",
|
|
21: "karty-lenorman-gora",
|
|
22: "karty-lenorman-razvilka",
|
|
23: "karty-lenorman-krysy",
|
|
24: "karta-lenorman-serdce",
|
|
25: "karty-lenorman-kolco",
|
|
26: "karty-lenorman-kniga",
|
|
27: "karty-lenorman-pismo",
|
|
28: "karty-lenorman-muzhchina",
|
|
29: "karty-lenorman-zhenshhina",
|
|
30: "karty-lenorman-lilii",
|
|
31: "karty-lenorman-solnce",
|
|
32: "karty-lenorman-luna",
|
|
33: "karty-lenorman-klyuch",
|
|
34: "karty-lenorman-ryby",
|
|
35: "karty-lenorman-yakor",
|
|
36: "karty-lenorman-krest",
|
|
}
|
|
|
|
SOURCE_LABELS = {
|
|
"karusel_table": "Карусель · таблица",
|
|
"karusel_short": "Карусель · кратко",
|
|
"ezoplaneta": "Эзопланета",
|
|
"magya_meaning": "Magya · кратко",
|
|
"magya_pair": "Magya · эссе",
|
|
"karusel": "Карусель",
|
|
"magya": "Magya",
|
|
}
|
|
|
|
SOURCE_ORDER = [
|
|
"magya_pair",
|
|
"karusel_table",
|
|
"magya_meaning",
|
|
"ezoplaneta",
|
|
"karusel_short",
|
|
]
|
|
|
|
|
|
def sort_readings(readings: list[dict]) -> list[dict]:
|
|
rank = {s: i for i, s in enumerate(SOURCE_ORDER)}
|
|
return sorted(readings, key=lambda r: (rank.get(r.get("source"), 99), r.get("left", 0)))
|
|
|
|
|
|
def strip_tags(html: str) -> str:
|
|
html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
|
|
html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
|
|
html = re.sub(r"(?i)<br\s*/?>", "\n", html)
|
|
html = re.sub(r"(?i)</p>", "\n\n", html)
|
|
html = re.sub(r"(?i)</h[1-6]>", "\n", html)
|
|
html = re.sub(r"(?i)</div>", "\n", html)
|
|
html = re.sub(r"<[^>]+>", " ", html)
|
|
html = html.replace("\xa0", " ")
|
|
html = re.sub(r"[ \t]+", " ", html)
|
|
html = re.sub(r"\n[ \t]+", "\n", html)
|
|
html = re.sub(r"\n{3,}", "\n\n", html)
|
|
return html.strip()
|
|
|
|
|
|
def clean_magya_meaning(text: str) -> str:
|
|
m = re.search(r"Краткое значение:", text)
|
|
if m:
|
|
text = text[m.start() :]
|
|
text = re.split(r"(?i)Сочетание карты .+ с другими картами", text)[0]
|
|
text = re.sub(r"(?is)Гадания\s*>[\s\S]{0,200}?Всадник\s*", "", text)
|
|
text = re.sub(r"^\s*>\s*", "", text, flags=re.M)
|
|
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
|
|
|
|
|
def clean_ezo_meaning(text: str) -> str:
|
|
text = re.sub(r"(?im)^.*\.(jpg|png|jpeg).*$", "", text)
|
|
text = re.sub(r"(?im)^.*\d[\d\s.,]*\s*КБ.*просмотр.*$", "", text)
|
|
text = re.split(r"(?i)\nСочетани", text)[0]
|
|
text = re.sub(r"\n{3,}", "\n\n", text).strip()
|
|
return text
|
|
|
|
|
|
def clean_karusel_from_cache(n: int, fallback: str) -> str:
|
|
slug = KARUSEL_MEAN_SLUG.get(n)
|
|
if not slug:
|
|
return fallback
|
|
path = HTML_CACHE / f"karusel-magii.ru_{slug}_.html"
|
|
if not path.exists():
|
|
path = HTML_CACHE / f"karusel-magii.ru_{slug}.html"
|
|
if not path.exists():
|
|
return fallback
|
|
html = path.read_text(encoding="utf-8", errors="replace")
|
|
m = re.search(r'(?is)<div class="content clearfix">(.*)', html)
|
|
if not m:
|
|
return fallback
|
|
body = m.group(1)
|
|
for marker in ('id="sidebar', 'id="footer', "sharedaddy", "yarpp"):
|
|
i = body.find(marker)
|
|
if i > 200:
|
|
body = body[:i]
|
|
break
|
|
text = strip_tags(body)
|
|
text = re.split(r"(?i)Смотрите также", text)[0]
|
|
text = re.split(r"(?i)Рекомендуем по этой теме", text)[0]
|
|
text = re.sub(r"(?im)^Карты Ленорман .*$", "", text)
|
|
text = re.sub(r"\n{3,}", "\n\n", text).strip()
|
|
return text if len(text) > 80 else fallback
|
|
|
|
|
|
def preview_of(readings: list[dict]) -> str:
|
|
for src in SOURCE_ORDER:
|
|
for r in readings:
|
|
if r.get("source") == src and r.get("text"):
|
|
t = r["text"].replace("\n", " ").strip()
|
|
return t if len(t) <= 160 else t[:157].rstrip() + "…"
|
|
return ""
|
|
|
|
|
|
def main() -> None:
|
|
corpus = json.loads(CORPUS.read_text(encoding="utf-8"))
|
|
canon = json.loads(CANON.read_text(encoding="utf-8"))
|
|
unified = {}
|
|
if UNIFIED.exists():
|
|
unified = json.loads(UNIFIED.read_text(encoding="utf-8"))
|
|
stations = canon["lenormand"]["stations"]
|
|
by_n: dict[int, dict] = {}
|
|
for st in stations:
|
|
sid = st.get("id", "")
|
|
if sid.startswith("card_"):
|
|
n = int(sid.split("_")[1])
|
|
by_n[n] = st
|
|
|
|
cards_out = []
|
|
for n in range(1, 37):
|
|
cc = corpus["cards"][str(n)]
|
|
st = by_n.get(n, {})
|
|
meanings = []
|
|
magya_short = ""
|
|
for m in cc.get("meanings", []):
|
|
src = m.get("source", "")
|
|
text = (m.get("text") or "").strip()
|
|
if src == "magya":
|
|
text = clean_magya_meaning(text)
|
|
mm = re.search(r"Краткое значение:\s*(.+)", text)
|
|
if mm:
|
|
magya_short = mm.group(1).split("\n")[0].strip(" .")
|
|
elif src == "ezoplaneta":
|
|
text = clean_ezo_meaning(text)
|
|
elif src == "karusel":
|
|
text = clean_karusel_from_cache(n, text)
|
|
if len(text) < 40:
|
|
continue
|
|
meanings.append(
|
|
{
|
|
"source": src,
|
|
"label": SOURCE_LABELS.get(src, src),
|
|
"text": text,
|
|
}
|
|
)
|
|
pairs = []
|
|
for other in range(1, 37):
|
|
if other == n:
|
|
continue
|
|
key = f"{min(n, other):02d}-{max(n, other):02d}"
|
|
bucket = corpus["pairs"].get(key)
|
|
if not bucket:
|
|
continue
|
|
readings = []
|
|
for r in sort_readings(bucket.get("readings", [])):
|
|
readings.append(
|
|
{
|
|
"source": r.get("source"),
|
|
"label": SOURCE_LABELS.get(r.get("source", ""), r.get("source", "")),
|
|
"left": r.get("left"),
|
|
"right": r.get("right"),
|
|
"text": (r.get("text") or "").strip(),
|
|
}
|
|
)
|
|
if not readings:
|
|
continue
|
|
ost = by_n.get(other, {})
|
|
up = (unified.get("pairs") or {}).get(key) or {}
|
|
utext = (up.get("text") or "").strip()
|
|
pairs.append(
|
|
{
|
|
"n": other,
|
|
"id": f"card_{other}",
|
|
"name": corpus["cards"][str(other)]["name"],
|
|
"symbol": ost.get("symbol", ""),
|
|
"preview": (utext[:157] + "…") if len(utext) > 160 else (utext or preview_of(readings)),
|
|
"text": utext,
|
|
"readings": [] if utext else readings,
|
|
}
|
|
)
|
|
ucard = (unified.get("cards") or {}).get(str(n)) or {}
|
|
umeaning = (ucard.get("meaning") or "").strip()
|
|
ucore = (ucard.get("core") or "").strip()
|
|
cards_out.append(
|
|
{
|
|
"n": n,
|
|
"id": f"card_{n}",
|
|
"name": cc["name"],
|
|
"symbol": st.get("symbol", ""),
|
|
"lore": ucore or magya_short or st.get("lore", ""),
|
|
"group": st.get("group", f"Карта {n}"),
|
|
"meaning": umeaning,
|
|
"shadow": (ucard.get("shadow") or "").strip(),
|
|
"meanings": [] if umeaning else meanings,
|
|
"pairs": pairs,
|
|
}
|
|
)
|
|
|
|
names = {c["n"]: c["name"] for c in cards_out}
|
|
triads = []
|
|
uni_triads = unified.get("triads") or []
|
|
if uni_triads:
|
|
for t in uni_triads:
|
|
nums = [int(x) for x in t.get("cards", [])]
|
|
if len(nums) != 3:
|
|
continue
|
|
triads.append(
|
|
{
|
|
"cards": nums,
|
|
"names": [names.get(x, str(x)) for x in nums],
|
|
"text": (t.get("text") or "").strip(),
|
|
}
|
|
)
|
|
else:
|
|
for t in corpus.get("triads", []):
|
|
nums = [int(x) for x in t.get("cards", []) if str(x).isdigit()]
|
|
if len(nums) != 3:
|
|
continue
|
|
triads.append(
|
|
{
|
|
"cards": nums,
|
|
"names": [names.get(x, str(x)) for x in nums],
|
|
"text": (t.get("text") or "").strip(),
|
|
}
|
|
)
|
|
|
|
extras = []
|
|
for e in corpus.get("extras", []):
|
|
title = (e.get("title") or "").strip()
|
|
text = (e.get("text") or "").strip()
|
|
if len(text) < 80:
|
|
continue
|
|
extras.append(
|
|
{
|
|
"source": e.get("source", ""),
|
|
"title": title,
|
|
"text": text,
|
|
}
|
|
)
|
|
|
|
blob = {
|
|
"system": "lenormand",
|
|
"source_labels": SOURCE_LABELS,
|
|
"card_count": 36,
|
|
"pair_count": sum(len(c["pairs"]) for c in cards_out) // 2,
|
|
"cards": cards_out,
|
|
"triads": triads,
|
|
"extras": extras,
|
|
}
|
|
text = json.dumps(blob, ensure_ascii=False, indent=2) + "\n"
|
|
OUT_ASSET.parent.mkdir(parents=True, exist_ok=True)
|
|
OUT_ASSET.write_text(text, encoding="utf-8")
|
|
OUT_DATA.write_text(text, encoding="utf-8")
|
|
|
|
# Supplement station lore in traditions canon (short line on the card).
|
|
lore_by_n = {c["n"]: c["lore"] for c in cards_out if c.get("lore")}
|
|
for canon_path in (CANON, ASSET_CANON):
|
|
data = json.loads(canon_path.read_text(encoding="utf-8"))
|
|
ln = data["lenormand"]
|
|
ln["description"] = (
|
|
"36 карт французского оракула мадемуазель Ленорман: значения с трёх источников "
|
|
"и 630 парных сочетаний (карусель, эзопланета, magya) — источники не схлопываются."
|
|
)
|
|
for st in ln["stations"]:
|
|
sid = st.get("id", "")
|
|
if not sid.startswith("card_"):
|
|
continue
|
|
n = int(sid.split("_")[1])
|
|
if lore_by_n.get(n):
|
|
st["lore"] = lore_by_n[n]
|
|
canon_path.write_text(
|
|
json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
|
|
if HTML_VIEWER.exists():
|
|
html = HTML_VIEWER.read_text(encoding="utf-8")
|
|
start = html.find(" const TRADITIONS_CANON = ")
|
|
end = html.find(" const WORDS_DATA = ")
|
|
if start > 0 and end > start:
|
|
data = json.loads(CANON.read_text(encoding="utf-8"))
|
|
blob_js = (
|
|
" const TRADITIONS_CANON = "
|
|
+ json.dumps(data, ensure_ascii=False, indent=2)
|
|
+ ";\n\n"
|
|
)
|
|
HTML_VIEWER.write_text(html[:start] + blob_js + html[end:], encoding="utf-8")
|
|
print("patched", HTML_VIEWER.relative_to(ROOT))
|
|
|
|
print(
|
|
f"cards={len(cards_out)} pairs_per_card={len(cards_out[0]['pairs'])} "
|
|
f"triads={len(triads)} extras={len(extras)} bytes={len(text)}"
|
|
)
|
|
print("wrote", OUT_ASSET.relative_to(ROOT))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|