204 lines
7.2 KiB
Python
204 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
||
"""Reparse cached Lenormand HTML: drop magya index-page false pairs, recover ezo combos."""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import sys
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "scripts"))
|
||
from scrape_lenormand_sources import ( # noqa: E402
|
||
ALIAS_TO_NUM,
|
||
BY_NUM,
|
||
CORPUS_PATH,
|
||
COV_PATH,
|
||
HTML_DIR,
|
||
MAGYA_SLUG,
|
||
add_pair,
|
||
coverage,
|
||
extract_divs_by_class,
|
||
magya_main,
|
||
resolve_name,
|
||
strip_tags,
|
||
)
|
||
|
||
INDEX_MARKERS = (
|
||
"друг с другом и значение",
|
||
"выбрать тот аркан",
|
||
"все возможные сочетания карт ленорман",
|
||
)
|
||
|
||
|
||
CARD_TITLE_RE = re.compile(
|
||
r"^\s*(?:Гадания\s*>\s*)?(?:Значение карт Ленорман\s*>\s*)?"
|
||
r"(?:Сочетание карт Ленорман\s*>\s*)?"
|
||
r"(?:[А-ЯЁа-яё]{3,20}\s+и\s+[А-ЯЁа-яё]{3,20}\s*)?",
|
||
re.I,
|
||
)
|
||
|
||
|
||
def clean_magya_pair_text(html: str) -> str:
|
||
text = magya_main(html)
|
||
text = re.sub(r"(?is)Смотрите также:[\s\S]+$", "", text).strip()
|
||
text = CARD_TITLE_RE.sub("", text, count=1)
|
||
text = re.sub(r"\s+", " ", text).strip(" >\n\t")
|
||
return text
|
||
|
||
|
||
def is_index_text(text: str) -> bool:
|
||
t = text.lower()
|
||
return any(m in t for m in INDEX_MARKERS)
|
||
|
||
|
||
def _clean_meaning(text: str) -> str:
|
||
text = re.sub(r"^\s*[-—–:]\s*", "", text)
|
||
lines = [ln.strip(" -—–") for ln in text.split("\n") if ln.strip(" -—–")]
|
||
if not lines:
|
||
return ""
|
||
meaning = lines[0]
|
||
if meaning.lower().startswith(("jpg", "png", "просмотр")):
|
||
return ""
|
||
return meaning
|
||
|
||
|
||
def ezo_combos_from_post(raw: str, left: int) -> list[tuple[int, str]]:
|
||
chunk = raw
|
||
m = re.search(r"(?i)Сочетани[яе]", raw)
|
||
if m:
|
||
chunk = raw[m.end() :]
|
||
found: dict[int, str] = {}
|
||
parts = re.split(r"<strong[^>]*>\s*([^<]+?)\s*</strong>", chunk)
|
||
for i in range(1, len(parts) - 1, 2):
|
||
right = resolve_name(parts[i])
|
||
if not right or right == left:
|
||
continue
|
||
meaning = _clean_meaning(strip_tags(parts[i + 1]))
|
||
if len(meaning) < 4:
|
||
continue
|
||
found[right] = meaning
|
||
# list form: "- Всадник: подстрекательство."
|
||
plain = strip_tags(chunk)
|
||
for mm in re.finditer(
|
||
r"(?m)^\s*[-—–]?\s*([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-]{2,20})\s*[:—–-]\s*(.+)$",
|
||
plain,
|
||
):
|
||
right = resolve_name(mm.group(1))
|
||
if not right or right == left or right in found:
|
||
continue
|
||
meaning = mm.group(2).strip()
|
||
if len(meaning) < 4:
|
||
continue
|
||
found[right] = meaning
|
||
return list(found.items())
|
||
|
||
|
||
def main() -> None:
|
||
store = json.loads(CORPUS_PATH.read_text(encoding="utf-8"))
|
||
|
||
# Drop magya_pair readings that are actually the section index.
|
||
dropped = 0
|
||
for bucket in store["pairs"].values():
|
||
keep = []
|
||
for r in bucket["readings"]:
|
||
if r["source"] == "magya_pair" and is_index_text(r["text"]):
|
||
dropped += 1
|
||
continue
|
||
keep.append(r)
|
||
bucket["readings"] = keep
|
||
print(f"dropped magya index-false pairs: {dropped}")
|
||
|
||
# Rebuild magya_pair texts from cache with a tighter title strip.
|
||
rebuilt = 0
|
||
slug_to_num = {v: k for k, v in MAGYA_SLUG.items()}
|
||
for html_path in HTML_DIR.glob("magya-online.ru_sochetanie_kart_lenorman_*_i_*.html"):
|
||
name = html_path.stem.replace("magya-online.ru_sochetanie_kart_lenorman_", "")
|
||
if "_i_" not in name:
|
||
continue
|
||
a_slug, b_slug = name.split("_i_", 1)
|
||
a = slug_to_num.get(a_slug)
|
||
b = slug_to_num.get(b_slug)
|
||
if not a or not b:
|
||
continue
|
||
html = html_path.read_text(encoding="utf-8", errors="replace")
|
||
text = clean_magya_pair_text(html)
|
||
if is_index_text(text) or len(text) < 80:
|
||
continue
|
||
url = f"https://magya-online.ru/sochetanie_kart_lenorman/{a_slug}_i_{b_slug}"
|
||
# replace existing magya_pair for this direction
|
||
key = f"{min(a,b):02d}-{max(a,b):02d}"
|
||
bucket = store["pairs"].setdefault(key, {"a": min(a, b), "b": max(a, b), "readings": []})
|
||
bucket["readings"] = [
|
||
r
|
||
for r in bucket["readings"]
|
||
if not (r["source"] == "magya_pair" and r.get("left") == a and r.get("right") == b)
|
||
]
|
||
add_pair(store, a, b, "magya_pair", url, text, "a_then_b")
|
||
rebuilt += 1
|
||
print(f"rebuilt magya_pair essays: {rebuilt}")
|
||
|
||
# Re-extract ezo combos from cached thread pages.
|
||
ezo_added = 0
|
||
for html_path in sorted(HTML_DIR.glob("ezoplaneta.ru_viewtopic.php_t_1913*.html")):
|
||
html = html_path.read_text(encoding="utf-8", errors="replace")
|
||
url = "https://ezoplaneta.ru/viewtopic.php?t=1913"
|
||
posts = extract_divs_by_class(html, "content")
|
||
for raw in posts:
|
||
text = strip_tags(raw)
|
||
head_html = re.search(
|
||
r'color:#008000["\']?[^>]*>\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][^<(]{1,40})',
|
||
raw,
|
||
)
|
||
left = None
|
||
if head_html:
|
||
left = int(head_html.group(1)) if head_html.group(1) else resolve_name(
|
||
head_html.group(2)
|
||
)
|
||
if not isinstance(left, int) or not (1 <= left <= 36):
|
||
title = re.sub(r"\s+", " ", text.split("\n", 1)[0])[:90]
|
||
if len(text) > 400 and not any(
|
||
e.get("title") == title for e in store["extras"]
|
||
):
|
||
store["extras"].append(
|
||
{
|
||
"source": "ezoplaneta",
|
||
"url": url,
|
||
"title": title,
|
||
"text": text,
|
||
}
|
||
)
|
||
continue
|
||
for right, meaning in ezo_combos_from_post(raw, left):
|
||
before = len(store["pairs"].get(f"{min(left,right):02d}-{max(left,right):02d}", {}).get("readings", []))
|
||
add_pair(store, left, right, "ezoplaneta", url, meaning, "a_then_b")
|
||
after = len(store["pairs"].get(f"{min(left,right):02d}-{max(left,right):02d}", {}).get("readings", []))
|
||
if after > before:
|
||
ezo_added += 1
|
||
print(f"ezo combos newly added: {ezo_added}")
|
||
|
||
store["meta"]["pair_count"] = len(store["pairs"])
|
||
store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
|
||
CORPUS_PATH.write_text(
|
||
json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
cov = coverage(store)
|
||
COV_PATH.write_text(cov, encoding="utf-8")
|
||
print(cov)
|
||
|
||
# ezo leftover report
|
||
from collections import Counter
|
||
left = Counter()
|
||
for b in store["pairs"].values():
|
||
for r in b["readings"]:
|
||
if r["source"] == "ezoplaneta":
|
||
left[r["left"]] += 1
|
||
print("ezo directed by card:")
|
||
for n in range(1, 37):
|
||
print(f" {n:02d} {BY_NUM[n][1]}: {left[n]}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|