feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets
This commit is contained in:
@@ -0,0 +1,748 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scrape Lenormand meanings + pair combinations from three Russian sources.
|
||||
|
||||
Sources (kept separate — they disagree):
|
||||
karusel https://karusel-magii.ru/
|
||||
ezoplaneta https://ezoplaneta.ru/viewtopic.php?t=1913
|
||||
magya https://magya-online.ru/
|
||||
|
||||
Writes:
|
||||
oracle_enricher/data/sources/lenormand_corpus.json
|
||||
oracle_enricher/out/lenormand_scrape/coverage.md
|
||||
oracle_enricher/out/lenormand_scrape/html/ (raw cache)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html as htmlmod
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT_DIR = ROOT / "oracle_enricher" / "out" / "lenormand_scrape"
|
||||
HTML_DIR = OUT_DIR / "html"
|
||||
CORPUS_PATH = ROOT / "oracle_enricher" / "data" / "sources" / "lenormand_corpus.json"
|
||||
COV_PATH = OUT_DIR / "coverage.md"
|
||||
|
||||
CTX = ssl.create_default_context()
|
||||
UA = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
)
|
||||
SLEEP = 0.12
|
||||
|
||||
# number, id, canonical RU, aliases (lowercase)
|
||||
CARDS = [
|
||||
(1, "rider", "Всадник", ["всадник", "гонец"]),
|
||||
(2, "clover", "Клевер", ["клевер"]),
|
||||
(3, "ship", "Корабль", ["корабль"]),
|
||||
(4, "house", "Дом", ["дом"]),
|
||||
(5, "tree", "Дерево", ["дерево"]),
|
||||
(6, "clouds", "Тучи", ["тучи", "облака", "туча", "облако"]),
|
||||
(7, "snake", "Змея", ["змея"]),
|
||||
(8, "coffin", "Гроб", ["гроб"]),
|
||||
(9, "bouquet", "Букет", ["букет"]),
|
||||
(10, "scythe", "Коса", ["коса"]),
|
||||
(11, "whip", "Метла", ["метла", "метлы", "розги"]),
|
||||
(12, "birds", "Птицы", ["птицы", "совы", "сова", "птица"]),
|
||||
(13, "child", "Ребенок", ["ребенок", "ребёнок", "дитя"]),
|
||||
(14, "fox", "Лиса", ["лиса"]),
|
||||
(15, "bear", "Медведь", ["медведь"]),
|
||||
(16, "stars", "Звезды", ["звезды", "звезда", "звёзды"]),
|
||||
(17, "stork", "Аист", ["аист", "аисты"]),
|
||||
(18, "dog", "Собака", ["собака"]),
|
||||
(19, "tower", "Башня", ["башня"]),
|
||||
(20, "garden", "Сад", ["сад", "парк"]),
|
||||
(21, "mountain", "Гора", ["гора"]),
|
||||
(22, "crossroads", "Развилка", ["развилка", "перекресток", "перекрёсток", "развилки"]),
|
||||
(23, "mice", "Крысы", ["крысы", "крыса", "мыши", "мышь"]),
|
||||
(24, "heart", "Сердце", ["сердце"]),
|
||||
(25, "ring", "Кольцо", ["кольцо"]),
|
||||
(26, "book", "Книга", ["книга"]),
|
||||
(27, "letter", "Письмо", ["письмо"]),
|
||||
(28, "man", "Мужчина", ["мужчина", "барин"]),
|
||||
(29, "woman", "Женщина", ["женщина", "барыня"]),
|
||||
(30, "lilies", "Лилии", ["лилии", "лилия"]),
|
||||
(31, "sun", "Солнце", ["солнце"]),
|
||||
(32, "moon", "Луна", ["луна"]),
|
||||
(33, "key", "Ключ", ["ключ"]),
|
||||
(34, "fish", "Рыбы", ["рыбы", "рыба"]),
|
||||
(35, "anchor", "Якорь", ["якорь"]),
|
||||
(36, "cross", "Крест", ["крест"]),
|
||||
]
|
||||
BY_NUM = {n: (i, name, aliases) for n, i, name, aliases in CARDS}
|
||||
ALIAS_TO_NUM: dict[str, int] = {}
|
||||
for n, _i, name, aliases in CARDS:
|
||||
ALIAS_TO_NUM[name.lower()] = n
|
||||
for a in aliases:
|
||||
ALIAS_TO_NUM[a] = n
|
||||
|
||||
MAGYA_SLUG = {
|
||||
1: "vsadnik", 2: "klever", 3: "korabl", 4: "dom", 5: "derevo",
|
||||
6: "tuchi", 7: "zmeya", 8: "grob", 9: "buket", 10: "kosa",
|
||||
11: "metla", 12: "sovi", 13: "rebenok", 14: "lisa", 15: "medved",
|
||||
16: "zvezdi", 17: "aist", 18: "sobaka", 19: "bashnya", 20: "sad",
|
||||
21: "gora", 22: "razvilka", 23: "krisi", 24: "serdce", 25: "kolco",
|
||||
26: "kniga", 27: "pismo", 28: "mujchina", 29: "jenschina", 30: "lilii",
|
||||
31: "solnce", 32: "luna", 33: "kluch", 34: "ribi", 35: "yakor", 36: "krest",
|
||||
}
|
||||
|
||||
KARUSEL_COMBO = {
|
||||
1: "karta-vsadnik-lenorman",
|
||||
2: "karta-klever-lenorman",
|
||||
3: "karta-korabl-lenorman",
|
||||
4: "karta-dom-lenorman",
|
||||
5: "karta-derevo-lenorman",
|
||||
6: "karta-oblaka-lenorman",
|
||||
7: "karta-zmeya-lenorman",
|
||||
8: "karta-grob-lenorman",
|
||||
9: "karta-buket-lenorman",
|
||||
10: "karta-kosa-lenorman",
|
||||
11: "karta-metla-lenorman",
|
||||
12: "karta-ptizi-lenorman",
|
||||
13: "karta-rebenok-lenorman",
|
||||
14: "karta-lisa-lenorman",
|
||||
15: "karta-medved-lenorman",
|
||||
16: "karta-zvezdy-lenorman",
|
||||
17: "karta-aist-lenorman",
|
||||
18: "karta-sobaka-lenorman",
|
||||
19: "karta-bashnya-lenorman",
|
||||
}
|
||||
|
||||
KARUSEL_MEAN = {
|
||||
1: "karty-lenorman-vsadnik",
|
||||
2: "karty-lenorman-klever",
|
||||
3: "karty-lenorman-korabl",
|
||||
4: "karty-lenorman-dom",
|
||||
5: "karty-lenorman-derevo",
|
||||
6: "karty-lenorman-tuchi",
|
||||
7: "karty-lenorman-zmeya",
|
||||
8: "karty-lenorman-grob",
|
||||
9: "karty-lenorman-buket",
|
||||
10: "karty-lenorman-kosa",
|
||||
11: "karty-lenorman-metla",
|
||||
12: "karty-lenorman-sovy",
|
||||
13: "karty-lenorman-rebenok",
|
||||
14: "karty-lenorman-lisa",
|
||||
15: "karty-lenorman-medved",
|
||||
16: "karty-lenorman-zvezdy",
|
||||
17: "karty-lenorman-aist",
|
||||
18: "karty-lenorman-sobaka",
|
||||
19: "karty-lenorman-bashnya",
|
||||
20: "karty-lenorman-sad",
|
||||
21: "karty-lenorman-gora",
|
||||
22: "karty-lenorman-razvilka",
|
||||
23: "karty-lenorman-krysy",
|
||||
24: "karta-lenorman-serdce",
|
||||
25: "karty-lenorman-kolco",
|
||||
26: "karty-lenorman-kniga",
|
||||
27: "karty-lenorman-pismo",
|
||||
28: "karty-lenorman-muzhchina",
|
||||
29: "karty-lenorman-zhenshhina",
|
||||
30: "karty-lenorman-lilii",
|
||||
31: "karty-lenorman-solnce",
|
||||
32: "karty-lenorman-luna",
|
||||
33: "karty-lenorman-klyuch",
|
||||
34: "karty-lenorman-ryby",
|
||||
35: "karty-lenorman-yakor",
|
||||
36: "karty-lenorman-krest",
|
||||
}
|
||||
|
||||
|
||||
def slugify_url(url: str) -> str:
|
||||
s = re.sub(r"^https?://", "", url)
|
||||
s = re.sub(r"[^a-zA-Z0-9._-]+", "_", s)
|
||||
return s[:180]
|
||||
|
||||
|
||||
def fetch(url: str, force: bool = False) -> tuple[int, str, str]:
|
||||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache = HTML_DIR / (slugify_url(url) + ".html")
|
||||
if cache.exists() and not force and cache.stat().st_size > 400:
|
||||
return 200, url, cache.read_text(encoding="utf-8", errors="replace")
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": UA,
|
||||
"Accept-Language": "ru,en;q=0.8",
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
|
||||
raw = resp.read()
|
||||
enc = resp.headers.get_content_charset() or "utf-8"
|
||||
text = raw.decode(enc, errors="replace")
|
||||
final = str(resp.url)
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
text = e.read().decode("utf-8", errors="replace") if e.fp else ""
|
||||
cache.write_text(text, encoding="utf-8")
|
||||
return e.code, url, text
|
||||
except Exception as e:
|
||||
return 0, url, f"ERROR {type(e).__name__}: {e}"
|
||||
cache.write_text(text, encoding="utf-8")
|
||||
time.sleep(SLEEP)
|
||||
return status, final, text
|
||||
|
||||
|
||||
def strip_tags(html: str) -> str:
|
||||
html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
|
||||
html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
|
||||
html = re.sub(r"(?is)<noscript[^>]*>.*?</noscript>", " ", html)
|
||||
html = re.sub(r"(?i)<br\s*/?>", "\n", html)
|
||||
html = re.sub(r"(?i)</p>", "\n\n", html)
|
||||
html = re.sub(r"(?i)</div>", "\n", html)
|
||||
html = re.sub(r"(?i)</li>", "\n", html)
|
||||
html = re.sub(r"(?i)</h[1-6]>", "\n", html)
|
||||
html = re.sub(r"(?i)</tr>", "\n", html)
|
||||
html = re.sub(r"(?i)</td>", " | ", html)
|
||||
html = re.sub(r"(?i)</th>", " | ", html)
|
||||
html = re.sub(r"<[^>]+>", " ", html)
|
||||
html = htmlmod.unescape(html)
|
||||
html = html.replace("\xa0", " ").replace(" ", " ")
|
||||
html = re.sub(r"[ \t]+", " ", html)
|
||||
html = re.sub(r"\n[ \t]+", "\n", html)
|
||||
html = re.sub(r"\n{3,}", "\n\n", html)
|
||||
return html.strip()
|
||||
|
||||
|
||||
def magya_main(html: str) -> str:
|
||||
m = re.search(r"(?is)<h1[^>]*>.*?</h1>(.*)", html)
|
||||
body = m.group(1) if m else html
|
||||
cut = re.search(
|
||||
r"(?is)(Добавить комментарий|Судьба и случай рекомендуют|Рейтинг:)",
|
||||
body,
|
||||
)
|
||||
if cut:
|
||||
body = body[: cut.start()]
|
||||
text = strip_tags(body)
|
||||
# drop leftover css
|
||||
text = re.sub(r"\.r_adapt_zag[\s\S]{0,400}?\}", "", text)
|
||||
text = re.sub(r"blockquote,[\s\S]{0,500}?inside;\s*\}", "", text)
|
||||
text = re.sub(r"li p \{[^}]+\}", "", text)
|
||||
text = re.sub(r"Гадания\s*>\s*.*?Всадник.*?\n", "", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def karusel_content(html: str) -> str:
|
||||
m = re.search(r'(?is)<div[^>]+class="[^"]*entry-content[^"]*"[^>]*>(.*)', html)
|
||||
body = m.group(1) if m else html
|
||||
cut = re.search(r'(?is)<div[^>]+class="[^"]*(sharedaddy|yarpp|comments)', body)
|
||||
if cut:
|
||||
body = body[: cut.start()]
|
||||
return strip_tags(body)
|
||||
|
||||
|
||||
def resolve_name(token: str) -> int | None:
|
||||
t = token.strip().lower()
|
||||
t = t.replace("ё", "е")
|
||||
t = re.sub(r"[^а-яa-z0-9]+", " ", t).strip()
|
||||
if not t:
|
||||
return None
|
||||
if t in ALIAS_TO_NUM:
|
||||
return ALIAS_TO_NUM[t]
|
||||
# first word
|
||||
w = t.split()[0]
|
||||
return ALIAS_TO_NUM.get(w)
|
||||
|
||||
|
||||
def pair_key(a: int, b: int) -> str:
|
||||
return f"{min(a, b):02d}-{max(a, b):02d}"
|
||||
|
||||
|
||||
def add_meaning(store, num: int, source: str, url: str, kind: str, text: str):
|
||||
text = text.strip()
|
||||
if len(text) < 40:
|
||||
return
|
||||
store["cards"][str(num)]["meanings"].append(
|
||||
{"source": source, "kind": kind, "url": url, "text": text}
|
||||
)
|
||||
|
||||
|
||||
def add_pair(store, a: int, b: int, source: str, url: str, text: str, order: str):
|
||||
if a == b:
|
||||
return
|
||||
text = re.sub(r"\s+", " ", text).strip(" -—–|")
|
||||
if len(text) < 8:
|
||||
return
|
||||
key = pair_key(a, b)
|
||||
bucket = store["pairs"].setdefault(
|
||||
key,
|
||||
{"a": min(a, b), "b": max(a, b), "readings": []},
|
||||
)
|
||||
# de-dupe identical source+text
|
||||
for r in bucket["readings"]:
|
||||
if r["source"] == source and r["text"] == text:
|
||||
return
|
||||
bucket["readings"].append(
|
||||
{
|
||||
"source": source,
|
||||
"url": url,
|
||||
"order": order, # a_then_b | undirected
|
||||
"left": a,
|
||||
"right": b,
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def parse_karusel_combo_table(html: str, left: int, url: str, store) -> int:
|
||||
n = 0
|
||||
# rows: number | NAME | meaning
|
||||
for m in re.finditer(
|
||||
r"(?is)<tr[^>]*>\s*<td[^>]*>\s*(\d{1,2})\s*</td>\s*<td[^>]*>\s*([^<]+)</td>\s*<td[^>]*>(.*?)</td>",
|
||||
html,
|
||||
):
|
||||
right = int(m.group(1))
|
||||
meaning = strip_tags(m.group(3))
|
||||
if right == left or right < 1 or right > 36:
|
||||
continue
|
||||
add_pair(store, left, right, "karusel_table", url, meaning, "a_then_b")
|
||||
n += 1
|
||||
if n:
|
||||
return n
|
||||
# markdown-like leftover or plain | 1 | ВСАДНИК | text |
|
||||
text = strip_tags(html)
|
||||
for m in re.finditer(
|
||||
r"(?m)^\s*(\d{1,2})\s*\|\s*([А-ЯЁа-яёA-Za-z\s]+)\s*\|\s*(.+)$",
|
||||
text,
|
||||
):
|
||||
right = int(m.group(1))
|
||||
meaning = m.group(3).strip()
|
||||
if right == left or right < 1 or right > 36:
|
||||
continue
|
||||
add_pair(store, left, right, "karusel_table", url, meaning, "a_then_b")
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def parse_karusel_short_pairs(html: str, url: str, store) -> int:
|
||||
text = karusel_content(html)
|
||||
n = 0
|
||||
current_left = None
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
m_head = re.match(
|
||||
r"^(?:#{0,6}\s*)?(\d{1,2})\.\s*([А-ЯЁа-яёA-Za-z]+)\s*\+",
|
||||
line,
|
||||
)
|
||||
if m_head:
|
||||
current_left = int(m_head.group(1))
|
||||
continue
|
||||
m_pair = re.match(
|
||||
r"^(\d{1,2})\s+([А-ЯЁа-яёA-Za-z\-]+)\s*[—–\-]\s*(.+)$",
|
||||
line,
|
||||
)
|
||||
if m_pair and current_left:
|
||||
right = int(m_pair.group(1))
|
||||
add_pair(
|
||||
store,
|
||||
current_left,
|
||||
right,
|
||||
"karusel_short",
|
||||
url,
|
||||
m_pair.group(3),
|
||||
"a_then_b",
|
||||
)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def parse_karusel_triads(html: str, url: str, store) -> int:
|
||||
text = karusel_content(html)
|
||||
n = 0
|
||||
for m in re.finditer(
|
||||
r"(\d{1,2})\s*\+\s*(\d{1,2})\s*\+\s*(\d{1,2})\s*=\s*(.+)",
|
||||
text,
|
||||
):
|
||||
a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
if not all(1 <= x <= 36 for x in (a, b, c)):
|
||||
continue
|
||||
store["triads"].append(
|
||||
{
|
||||
"cards": [a, b, c],
|
||||
"source": "karusel_triad",
|
||||
"url": url,
|
||||
"text": m.group(4).strip(),
|
||||
}
|
||||
)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def extract_divs_by_class(html: str, cls: str) -> list[str]:
|
||||
"""Return inner HTML of each <div class="cls"> with nested divs balanced."""
|
||||
pat = re.compile(rf'<div\s+class="{re.escape(cls)}">', re.I)
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
while True:
|
||||
m = pat.search(html, i)
|
||||
if not m:
|
||||
break
|
||||
pos = m.end()
|
||||
depth = 1
|
||||
j = pos
|
||||
captured = None
|
||||
while j < len(html) and depth:
|
||||
nxt_open = html.find("<div", j)
|
||||
nxt_close = html.find("</div>", j)
|
||||
if nxt_close < 0:
|
||||
break
|
||||
if nxt_open != -1 and nxt_open < nxt_close:
|
||||
depth += 1
|
||||
j = nxt_open + 4
|
||||
else:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
captured = html[pos:nxt_close]
|
||||
i = nxt_close + 6
|
||||
break
|
||||
j = nxt_close + 6
|
||||
if captured is None:
|
||||
break
|
||||
out.append(captured)
|
||||
return out
|
||||
|
||||
|
||||
def parse_ezo_thread(html: str, url: str, store) -> dict:
|
||||
stats = {"meanings": 0, "pairs": 0, "extras": 0}
|
||||
posts = extract_divs_by_class(html, "content")
|
||||
extra_re = re.compile(
|
||||
r"(?i)перевёрнут|карта дня|кратк\w+ значен|комбинац|магическ|кармическ|финансов|любви|работ"
|
||||
)
|
||||
for raw in posts:
|
||||
text = strip_tags(raw)
|
||||
current_left = None
|
||||
head = re.search(
|
||||
r"(?i)(?:^|\n)\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\s]{1,24})\s*(?:\(|$)",
|
||||
text,
|
||||
)
|
||||
if not head:
|
||||
head_html = re.search(
|
||||
r'color:#008000["\']?[^>]*>\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][^<(]{1,40})',
|
||||
raw,
|
||||
)
|
||||
if head_html:
|
||||
class _H:
|
||||
def group(self, i):
|
||||
return head_html.group(i)
|
||||
head = _H()
|
||||
if head:
|
||||
num = int(head.group(1)) if head.group(1) else resolve_name(head.group(2))
|
||||
if isinstance(num, int) and 1 <= num <= 36:
|
||||
current_left = num
|
||||
add_meaning(store, num, "ezoplaneta", url, "card", text)
|
||||
stats["meanings"] += 1
|
||||
elif extra_re.search(text[:400]) and len(text) > 200:
|
||||
title = re.sub(r"\s+", " ", text.split("\n", 1)[0])[:90]
|
||||
store["extras"].append(
|
||||
{"source": "ezoplaneta", "url": url, "title": title, "text": text}
|
||||
)
|
||||
stats["extras"] += 1
|
||||
|
||||
if not current_left:
|
||||
continue
|
||||
# Early posts: <strong>КЛЕВЕР</strong>-Скорое...
|
||||
# Later posts: <strong>Всадник</strong><br>- Неожиданная...
|
||||
for m in re.finditer(
|
||||
r"<strong[^>]*>\s*([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-\s]{2,24}?)\s*</strong>"
|
||||
r"\s*(?:<br\s*/?>|\s)*[-—–:]?\s*([^<]+)",
|
||||
raw,
|
||||
):
|
||||
right = resolve_name(m.group(1))
|
||||
meaning = m.group(2).strip()
|
||||
if not right or right == current_left:
|
||||
continue
|
||||
if len(meaning) < 4:
|
||||
continue
|
||||
add_pair(
|
||||
store,
|
||||
current_left,
|
||||
right,
|
||||
"ezoplaneta",
|
||||
url,
|
||||
meaning,
|
||||
"a_then_b",
|
||||
)
|
||||
stats["pairs"] += 1
|
||||
return stats
|
||||
|
||||
|
||||
def parse_magya_meaning(html: str, num: int, url: str, store) -> int:
|
||||
text = magya_main(html)
|
||||
# split off combinations section
|
||||
parts = re.split(r"(?i)Сочетание карты .+? с другими картами", text, maxsplit=1)
|
||||
main = parts[0].strip()
|
||||
add_meaning(store, num, "magya", url, "card", main)
|
||||
n = 0
|
||||
if len(parts) > 1:
|
||||
combo_block = parts[1]
|
||||
for m in re.finditer(
|
||||
r"(?m)^([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-]{2,20})\s*[—–\-]\s*(.+)$",
|
||||
combo_block,
|
||||
):
|
||||
right = resolve_name(m.group(1))
|
||||
if not right or right == num:
|
||||
continue
|
||||
add_pair(store, num, right, "magya_meaning", url, m.group(2), "a_then_b")
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def parse_magya_pair(html: str, url: str, store) -> bool:
|
||||
m = re.search(r"/sochetanie_kart_lenorman/([a-z]+)_i_([a-z]+)", url)
|
||||
if not m:
|
||||
return False
|
||||
slug_a, slug_b = m.group(1), m.group(2)
|
||||
inv = {v: k for k, v in MAGYA_SLUG.items()}
|
||||
a = inv.get(slug_a)
|
||||
b = inv.get(slug_b)
|
||||
if not a or not b:
|
||||
return False
|
||||
text = magya_main(html)
|
||||
text = re.sub(r"(?is)Смотрите также:[\s\S]+$", "", text).strip()
|
||||
text = re.sub(r"(?is)Гадания\s*>[\s\S]{0,250}?\n+", "", text)
|
||||
text = re.sub(
|
||||
r"(?is)^(?:Значение карт Ленорман\s*>\s*)?(?:Сочетание карт Ленорман\s*>\s*)?[А-ЯЁа-яёA-Za-z\s]+и [А-ЯЁа-яёA-Za-z\s]+\s*",
|
||||
"",
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
if len(text) < 40:
|
||||
return False
|
||||
add_pair(store, a, b, "magya_pair", url, text, "a_then_b")
|
||||
return True
|
||||
|
||||
|
||||
def empty_store() -> dict:
|
||||
cards = {}
|
||||
for n, cid, name, aliases in CARDS:
|
||||
cards[str(n)] = {
|
||||
"id": cid,
|
||||
"number": n,
|
||||
"name": name,
|
||||
"aliases": aliases,
|
||||
"meanings": [],
|
||||
}
|
||||
return {
|
||||
"meta": {
|
||||
"system": "lenormand",
|
||||
"cards": 36,
|
||||
"sources": {
|
||||
"karusel": "https://karusel-magii.ru/",
|
||||
"ezoplaneta": "https://ezoplaneta.ru/viewtopic.php?t=1913",
|
||||
"magya": "https://magya-online.ru/",
|
||||
},
|
||||
"note": "Источники расходятся: readings[] хранит все версии без схлопывания.",
|
||||
},
|
||||
"cards": cards,
|
||||
"pairs": {},
|
||||
"triads": [],
|
||||
"extras": [],
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
|
||||
def scrape(limit_pairs: int | None = None) -> dict:
|
||||
store = empty_store()
|
||||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ---------- KARUSEL ----------
|
||||
print("== karusel meanings ==")
|
||||
for n, slug in KARUSEL_MEAN.items():
|
||||
url = f"https://karusel-magii.ru/{slug}/"
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
print(f" FAIL {n} {slug} {st}")
|
||||
continue
|
||||
text = karusel_content(html)
|
||||
add_meaning(store, n, "karusel", final, "card", text)
|
||||
print(f" mean {n:02d} {BY_NUM[n][1]} chars={len(text)}")
|
||||
|
||||
url = "https://karusel-magii.ru/znachenie-kart-lenorman/"
|
||||
st, final, html = fetch(url)
|
||||
if st == 200:
|
||||
add_meaning(store, 1, "karusel", final, "hub_skipped", "") # noop
|
||||
# keep as extra hub
|
||||
store["extras"].append(
|
||||
{
|
||||
"source": "karusel",
|
||||
"url": final,
|
||||
"title": "Значение карт Ленорман (сводная)",
|
||||
"text": karusel_content(html)[:20000],
|
||||
}
|
||||
)
|
||||
|
||||
print("== karusel combo tables ==")
|
||||
for n, slug in KARUSEL_COMBO.items():
|
||||
url = f"https://karusel-magii.ru/{slug}/"
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
print(f" FAIL combo {n} {st}")
|
||||
continue
|
||||
k = parse_karusel_combo_table(html, n, final, store)
|
||||
print(f" table {n:02d} {BY_NUM[n][1]} pairs={k}")
|
||||
|
||||
print("== karusel short pairs + triads ==")
|
||||
url = "https://karusel-magii.ru/znachenie-i-sochetanie-kart-lenorman/"
|
||||
st, final, html = fetch(url)
|
||||
if st == 200:
|
||||
k = parse_karusel_short_pairs(html, final, store)
|
||||
print(f" short pairs {k}")
|
||||
url = "https://karusel-magii.ru/sochetanie-kart-lenorman/"
|
||||
st, final, html = fetch(url)
|
||||
if st == 200:
|
||||
t = parse_karusel_triads(html, final, store)
|
||||
print(f" triads {t}")
|
||||
|
||||
# ---------- EZOPLANETA ----------
|
||||
print("== ezoplaneta thread ==")
|
||||
for start in (0, 10, 20, 30, 40):
|
||||
url = (
|
||||
"https://ezoplaneta.ru/viewtopic.php?t=1913"
|
||||
if start == 0
|
||||
else f"https://ezoplaneta.ru/viewtopic.php?t=1913&start={start}"
|
||||
)
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
print(f" FAIL start={start} {st}")
|
||||
continue
|
||||
stats = parse_ezo_thread(html, final, store)
|
||||
print(f" page start={start} {stats}")
|
||||
|
||||
# ---------- MAGYA meanings ----------
|
||||
print("== magya meanings ==")
|
||||
pair_urls: list[str] = []
|
||||
for n, slug in MAGYA_SLUG.items():
|
||||
url = f"https://magya-online.ru/znachenie_kart_lenorman/{slug}"
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
print(f" FAIL mean {slug} {st}")
|
||||
continue
|
||||
k = parse_magya_meaning(html, n, final, store)
|
||||
print(f" mean {n:02d} {slug} combos={k}")
|
||||
|
||||
hub = f"https://magya-online.ru/sochetanie_kart_lenorman/{slug}"
|
||||
st2, final2, html2 = fetch(hub)
|
||||
if st2 == 200:
|
||||
found = re.findall(
|
||||
rf"""href=['"](/sochetanie_kart_lenorman/{slug}_i_[a-z]+)['"]""",
|
||||
html2,
|
||||
)
|
||||
for rel in found:
|
||||
pair_urls.append("https://magya-online.ru" + rel)
|
||||
|
||||
# also generate directed slugs — magya uses cardA_i_cardB
|
||||
for a, sa in MAGYA_SLUG.items():
|
||||
for b, sb in MAGYA_SLUG.items():
|
||||
if a == b:
|
||||
continue
|
||||
pair_urls.append(
|
||||
f"https://magya-online.ru/sochetanie_kart_lenorman/{sa}_i_{sb}"
|
||||
)
|
||||
|
||||
# unique preserve order
|
||||
seen = set()
|
||||
uniq = []
|
||||
for u in pair_urls:
|
||||
if u not in seen:
|
||||
seen.add(u)
|
||||
uniq.append(u)
|
||||
if limit_pairs:
|
||||
uniq = uniq[:limit_pairs]
|
||||
print(f"== magya pair essays {len(uniq)} ==")
|
||||
ok = 0
|
||||
for i, url in enumerate(uniq, 1):
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
continue
|
||||
if parse_magya_pair(html, final, store):
|
||||
ok += 1
|
||||
if i % 50 == 0:
|
||||
print(f" pairs {i}/{len(uniq)} ok={ok}")
|
||||
print(f" pair essays ok={ok}/{len(uniq)}")
|
||||
|
||||
return store
|
||||
|
||||
|
||||
def coverage(store: dict) -> str:
|
||||
lines = [
|
||||
"# Lenormand corpus coverage",
|
||||
"",
|
||||
f"Карт: {len(store['cards'])}",
|
||||
f"Пар с хотя бы одним чтением: {len(store['pairs'])} / 630 undirected",
|
||||
f"Триад: {len(store['triads'])}",
|
||||
f"Доп. разделов: {len(store['extras'])}",
|
||||
f"Ошибок загрузки: {len(store['errors'])}",
|
||||
"",
|
||||
"## Значения по картам (число источников)",
|
||||
"",
|
||||
]
|
||||
for n in range(1, 37):
|
||||
c = store["cards"][str(n)]
|
||||
srcs = sorted({m["source"] for m in c["meanings"] if m.get("text")})
|
||||
lines.append(f"- {n:02d} {c['name']}: {len(c['meanings'])} записей ({', '.join(srcs) or '—'})")
|
||||
|
||||
by_src = defaultdict(int)
|
||||
disagree = 0
|
||||
for key, bucket in store["pairs"].items():
|
||||
srcs = {r["source"] for r in bucket["readings"]}
|
||||
for s in srcs:
|
||||
by_src[s] += 1
|
||||
if len(srcs) >= 2:
|
||||
disagree += 1
|
||||
lines += [
|
||||
"",
|
||||
"## Пары по источнику",
|
||||
"",
|
||||
]
|
||||
for s, n in sorted(by_src.items()):
|
||||
lines.append(f"- {s}: {n}")
|
||||
lines += [
|
||||
"",
|
||||
f"Пар, где источники расходятся (2+ источника): {disagree}",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--limit-pairs", type=int, default=None)
|
||||
ap.add_argument("--force", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if args.force:
|
||||
for p in HTML_DIR.glob("*.html"):
|
||||
p.unlink()
|
||||
store = scrape(limit_pairs=args.limit_pairs)
|
||||
store["meta"]["pair_count"] = len(store["pairs"])
|
||||
store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
|
||||
CORPUS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
CORPUS_PATH.write_text(
|
||||
json.dumps(store, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
cov = coverage(store)
|
||||
COV_PATH.write_text(cov, encoding="utf-8")
|
||||
print("\n" + cov)
|
||||
print("wrote", CORPUS_PATH)
|
||||
print("wrote", COV_PATH)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user