feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reparse cached Lenormand HTML: drop magya index-page false pairs, recover ezo combos."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from scrape_lenormand_sources import ( # noqa: E402
|
||||
ALIAS_TO_NUM,
|
||||
BY_NUM,
|
||||
CORPUS_PATH,
|
||||
COV_PATH,
|
||||
HTML_DIR,
|
||||
MAGYA_SLUG,
|
||||
add_pair,
|
||||
coverage,
|
||||
extract_divs_by_class,
|
||||
magya_main,
|
||||
resolve_name,
|
||||
strip_tags,
|
||||
)
|
||||
|
||||
INDEX_MARKERS = (
|
||||
"друг с другом и значение",
|
||||
"выбрать тот аркан",
|
||||
"все возможные сочетания карт ленорман",
|
||||
)
|
||||
|
||||
|
||||
CARD_TITLE_RE = re.compile(
|
||||
r"^\s*(?:Гадания\s*>\s*)?(?:Значение карт Ленорман\s*>\s*)?"
|
||||
r"(?:Сочетание карт Ленорман\s*>\s*)?"
|
||||
r"(?:[А-ЯЁа-яё]{3,20}\s+и\s+[А-ЯЁа-яё]{3,20}\s*)?",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def clean_magya_pair_text(html: str) -> str:
|
||||
text = magya_main(html)
|
||||
text = re.sub(r"(?is)Смотрите также:[\s\S]+$", "", text).strip()
|
||||
text = CARD_TITLE_RE.sub("", text, count=1)
|
||||
text = re.sub(r"\s+", " ", text).strip(" >\n\t")
|
||||
return text
|
||||
|
||||
|
||||
def is_index_text(text: str) -> bool:
|
||||
t = text.lower()
|
||||
return any(m in t for m in INDEX_MARKERS)
|
||||
|
||||
|
||||
def _clean_meaning(text: str) -> str:
|
||||
text = re.sub(r"^\s*[-—–:]\s*", "", text)
|
||||
lines = [ln.strip(" -—–") for ln in text.split("\n") if ln.strip(" -—–")]
|
||||
if not lines:
|
||||
return ""
|
||||
meaning = lines[0]
|
||||
if meaning.lower().startswith(("jpg", "png", "просмотр")):
|
||||
return ""
|
||||
return meaning
|
||||
|
||||
|
||||
def ezo_combos_from_post(raw: str, left: int) -> list[tuple[int, str]]:
|
||||
chunk = raw
|
||||
m = re.search(r"(?i)Сочетани[яе]", raw)
|
||||
if m:
|
||||
chunk = raw[m.end() :]
|
||||
found: dict[int, str] = {}
|
||||
parts = re.split(r"<strong[^>]*>\s*([^<]+?)\s*</strong>", chunk)
|
||||
for i in range(1, len(parts) - 1, 2):
|
||||
right = resolve_name(parts[i])
|
||||
if not right or right == left:
|
||||
continue
|
||||
meaning = _clean_meaning(strip_tags(parts[i + 1]))
|
||||
if len(meaning) < 4:
|
||||
continue
|
||||
found[right] = meaning
|
||||
# list form: "- Всадник: подстрекательство."
|
||||
plain = strip_tags(chunk)
|
||||
for mm in re.finditer(
|
||||
r"(?m)^\s*[-—–]?\s*([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-]{2,20})\s*[:—–-]\s*(.+)$",
|
||||
plain,
|
||||
):
|
||||
right = resolve_name(mm.group(1))
|
||||
if not right or right == left or right in found:
|
||||
continue
|
||||
meaning = mm.group(2).strip()
|
||||
if len(meaning) < 4:
|
||||
continue
|
||||
found[right] = meaning
|
||||
return list(found.items())
|
||||
|
||||
|
||||
def main() -> None:
|
||||
store = json.loads(CORPUS_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
# Drop magya_pair readings that are actually the section index.
|
||||
dropped = 0
|
||||
for bucket in store["pairs"].values():
|
||||
keep = []
|
||||
for r in bucket["readings"]:
|
||||
if r["source"] == "magya_pair" and is_index_text(r["text"]):
|
||||
dropped += 1
|
||||
continue
|
||||
keep.append(r)
|
||||
bucket["readings"] = keep
|
||||
print(f"dropped magya index-false pairs: {dropped}")
|
||||
|
||||
# Rebuild magya_pair texts from cache with a tighter title strip.
|
||||
rebuilt = 0
|
||||
slug_to_num = {v: k for k, v in MAGYA_SLUG.items()}
|
||||
for html_path in HTML_DIR.glob("magya-online.ru_sochetanie_kart_lenorman_*_i_*.html"):
|
||||
name = html_path.stem.replace("magya-online.ru_sochetanie_kart_lenorman_", "")
|
||||
if "_i_" not in name:
|
||||
continue
|
||||
a_slug, b_slug = name.split("_i_", 1)
|
||||
a = slug_to_num.get(a_slug)
|
||||
b = slug_to_num.get(b_slug)
|
||||
if not a or not b:
|
||||
continue
|
||||
html = html_path.read_text(encoding="utf-8", errors="replace")
|
||||
text = clean_magya_pair_text(html)
|
||||
if is_index_text(text) or len(text) < 80:
|
||||
continue
|
||||
url = f"https://magya-online.ru/sochetanie_kart_lenorman/{a_slug}_i_{b_slug}"
|
||||
# replace existing magya_pair for this direction
|
||||
key = f"{min(a,b):02d}-{max(a,b):02d}"
|
||||
bucket = store["pairs"].setdefault(key, {"a": min(a, b), "b": max(a, b), "readings": []})
|
||||
bucket["readings"] = [
|
||||
r
|
||||
for r in bucket["readings"]
|
||||
if not (r["source"] == "magya_pair" and r.get("left") == a and r.get("right") == b)
|
||||
]
|
||||
add_pair(store, a, b, "magya_pair", url, text, "a_then_b")
|
||||
rebuilt += 1
|
||||
print(f"rebuilt magya_pair essays: {rebuilt}")
|
||||
|
||||
# Re-extract ezo combos from cached thread pages.
|
||||
ezo_added = 0
|
||||
for html_path in sorted(HTML_DIR.glob("ezoplaneta.ru_viewtopic.php_t_1913*.html")):
|
||||
html = html_path.read_text(encoding="utf-8", errors="replace")
|
||||
url = "https://ezoplaneta.ru/viewtopic.php?t=1913"
|
||||
posts = extract_divs_by_class(html, "content")
|
||||
for raw in posts:
|
||||
text = strip_tags(raw)
|
||||
head_html = re.search(
|
||||
r'color:#008000["\']?[^>]*>\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][^<(]{1,40})',
|
||||
raw,
|
||||
)
|
||||
left = None
|
||||
if head_html:
|
||||
left = int(head_html.group(1)) if head_html.group(1) else resolve_name(
|
||||
head_html.group(2)
|
||||
)
|
||||
if not isinstance(left, int) or not (1 <= left <= 36):
|
||||
title = re.sub(r"\s+", " ", text.split("\n", 1)[0])[:90]
|
||||
if len(text) > 400 and not any(
|
||||
e.get("title") == title for e in store["extras"]
|
||||
):
|
||||
store["extras"].append(
|
||||
{
|
||||
"source": "ezoplaneta",
|
||||
"url": url,
|
||||
"title": title,
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
continue
|
||||
for right, meaning in ezo_combos_from_post(raw, left):
|
||||
before = len(store["pairs"].get(f"{min(left,right):02d}-{max(left,right):02d}", {}).get("readings", []))
|
||||
add_pair(store, left, right, "ezoplaneta", url, meaning, "a_then_b")
|
||||
after = len(store["pairs"].get(f"{min(left,right):02d}-{max(left,right):02d}", {}).get("readings", []))
|
||||
if after > before:
|
||||
ezo_added += 1
|
||||
print(f"ezo combos newly added: {ezo_added}")
|
||||
|
||||
store["meta"]["pair_count"] = len(store["pairs"])
|
||||
store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
|
||||
CORPUS_PATH.write_text(
|
||||
json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
cov = coverage(store)
|
||||
COV_PATH.write_text(cov, encoding="utf-8")
|
||||
print(cov)
|
||||
|
||||
# ezo leftover report
|
||||
from collections import Counter
|
||||
left = Counter()
|
||||
for b in store["pairs"].values():
|
||||
for r in b["readings"]:
|
||||
if r["source"] == "ezoplaneta":
|
||||
left[r["left"]] += 1
|
||||
print("ezo directed by card:")
|
||||
for n in range(1, 37):
|
||||
print(f" {n:02d} {BY_NUM[n][1]}: {left[n]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user