749 lines
25 KiB
Python
749 lines
25 KiB
Python
#!/usr/bin/env python3
|
||
"""Scrape Lenormand meanings + pair combinations from three Russian sources.
|
||
|
||
Sources (kept separate — they disagree):
|
||
karusel https://karusel-magii.ru/
|
||
ezoplaneta https://ezoplaneta.ru/viewtopic.php?t=1913
|
||
magya https://magya-online.ru/
|
||
|
||
Writes:
|
||
oracle_enricher/data/sources/lenormand_corpus.json
|
||
oracle_enricher/out/lenormand_scrape/coverage.md
|
||
oracle_enricher/out/lenormand_scrape/html/ (raw cache)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import html as htmlmod
|
||
import json
|
||
import re
|
||
import ssl
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OUT_DIR = ROOT / "oracle_enricher" / "out" / "lenormand_scrape"
|
||
HTML_DIR = OUT_DIR / "html"
|
||
CORPUS_PATH = ROOT / "oracle_enricher" / "data" / "sources" / "lenormand_corpus.json"
|
||
COV_PATH = OUT_DIR / "coverage.md"
|
||
|
||
CTX = ssl.create_default_context()
|
||
UA = (
|
||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||
)
|
||
SLEEP = 0.12
|
||
|
||
# number, id, canonical RU, aliases (lowercase)
|
||
CARDS = [
|
||
(1, "rider", "Всадник", ["всадник", "гонец"]),
|
||
(2, "clover", "Клевер", ["клевер"]),
|
||
(3, "ship", "Корабль", ["корабль"]),
|
||
(4, "house", "Дом", ["дом"]),
|
||
(5, "tree", "Дерево", ["дерево"]),
|
||
(6, "clouds", "Тучи", ["тучи", "облака", "туча", "облако"]),
|
||
(7, "snake", "Змея", ["змея"]),
|
||
(8, "coffin", "Гроб", ["гроб"]),
|
||
(9, "bouquet", "Букет", ["букет"]),
|
||
(10, "scythe", "Коса", ["коса"]),
|
||
(11, "whip", "Метла", ["метла", "метлы", "розги"]),
|
||
(12, "birds", "Птицы", ["птицы", "совы", "сова", "птица"]),
|
||
(13, "child", "Ребенок", ["ребенок", "ребёнок", "дитя"]),
|
||
(14, "fox", "Лиса", ["лиса"]),
|
||
(15, "bear", "Медведь", ["медведь"]),
|
||
(16, "stars", "Звезды", ["звезды", "звезда", "звёзды"]),
|
||
(17, "stork", "Аист", ["аист", "аисты"]),
|
||
(18, "dog", "Собака", ["собака"]),
|
||
(19, "tower", "Башня", ["башня"]),
|
||
(20, "garden", "Сад", ["сад", "парк"]),
|
||
(21, "mountain", "Гора", ["гора"]),
|
||
(22, "crossroads", "Развилка", ["развилка", "перекресток", "перекрёсток", "развилки"]),
|
||
(23, "mice", "Крысы", ["крысы", "крыса", "мыши", "мышь"]),
|
||
(24, "heart", "Сердце", ["сердце"]),
|
||
(25, "ring", "Кольцо", ["кольцо"]),
|
||
(26, "book", "Книга", ["книга"]),
|
||
(27, "letter", "Письмо", ["письмо"]),
|
||
(28, "man", "Мужчина", ["мужчина", "барин"]),
|
||
(29, "woman", "Женщина", ["женщина", "барыня"]),
|
||
(30, "lilies", "Лилии", ["лилии", "лилия"]),
|
||
(31, "sun", "Солнце", ["солнце"]),
|
||
(32, "moon", "Луна", ["луна"]),
|
||
(33, "key", "Ключ", ["ключ"]),
|
||
(34, "fish", "Рыбы", ["рыбы", "рыба"]),
|
||
(35, "anchor", "Якорь", ["якорь"]),
|
||
(36, "cross", "Крест", ["крест"]),
|
||
]
|
||
BY_NUM = {n: (i, name, aliases) for n, i, name, aliases in CARDS}
|
||
ALIAS_TO_NUM: dict[str, int] = {}
|
||
for n, _i, name, aliases in CARDS:
|
||
ALIAS_TO_NUM[name.lower()] = n
|
||
for a in aliases:
|
||
ALIAS_TO_NUM[a] = n
|
||
|
||
MAGYA_SLUG = {
|
||
1: "vsadnik", 2: "klever", 3: "korabl", 4: "dom", 5: "derevo",
|
||
6: "tuchi", 7: "zmeya", 8: "grob", 9: "buket", 10: "kosa",
|
||
11: "metla", 12: "sovi", 13: "rebenok", 14: "lisa", 15: "medved",
|
||
16: "zvezdi", 17: "aist", 18: "sobaka", 19: "bashnya", 20: "sad",
|
||
21: "gora", 22: "razvilka", 23: "krisi", 24: "serdce", 25: "kolco",
|
||
26: "kniga", 27: "pismo", 28: "mujchina", 29: "jenschina", 30: "lilii",
|
||
31: "solnce", 32: "luna", 33: "kluch", 34: "ribi", 35: "yakor", 36: "krest",
|
||
}
|
||
|
||
KARUSEL_COMBO = {
|
||
1: "karta-vsadnik-lenorman",
|
||
2: "karta-klever-lenorman",
|
||
3: "karta-korabl-lenorman",
|
||
4: "karta-dom-lenorman",
|
||
5: "karta-derevo-lenorman",
|
||
6: "karta-oblaka-lenorman",
|
||
7: "karta-zmeya-lenorman",
|
||
8: "karta-grob-lenorman",
|
||
9: "karta-buket-lenorman",
|
||
10: "karta-kosa-lenorman",
|
||
11: "karta-metla-lenorman",
|
||
12: "karta-ptizi-lenorman",
|
||
13: "karta-rebenok-lenorman",
|
||
14: "karta-lisa-lenorman",
|
||
15: "karta-medved-lenorman",
|
||
16: "karta-zvezdy-lenorman",
|
||
17: "karta-aist-lenorman",
|
||
18: "karta-sobaka-lenorman",
|
||
19: "karta-bashnya-lenorman",
|
||
}
|
||
|
||
KARUSEL_MEAN = {
|
||
1: "karty-lenorman-vsadnik",
|
||
2: "karty-lenorman-klever",
|
||
3: "karty-lenorman-korabl",
|
||
4: "karty-lenorman-dom",
|
||
5: "karty-lenorman-derevo",
|
||
6: "karty-lenorman-tuchi",
|
||
7: "karty-lenorman-zmeya",
|
||
8: "karty-lenorman-grob",
|
||
9: "karty-lenorman-buket",
|
||
10: "karty-lenorman-kosa",
|
||
11: "karty-lenorman-metla",
|
||
12: "karty-lenorman-sovy",
|
||
13: "karty-lenorman-rebenok",
|
||
14: "karty-lenorman-lisa",
|
||
15: "karty-lenorman-medved",
|
||
16: "karty-lenorman-zvezdy",
|
||
17: "karty-lenorman-aist",
|
||
18: "karty-lenorman-sobaka",
|
||
19: "karty-lenorman-bashnya",
|
||
20: "karty-lenorman-sad",
|
||
21: "karty-lenorman-gora",
|
||
22: "karty-lenorman-razvilka",
|
||
23: "karty-lenorman-krysy",
|
||
24: "karta-lenorman-serdce",
|
||
25: "karty-lenorman-kolco",
|
||
26: "karty-lenorman-kniga",
|
||
27: "karty-lenorman-pismo",
|
||
28: "karty-lenorman-muzhchina",
|
||
29: "karty-lenorman-zhenshhina",
|
||
30: "karty-lenorman-lilii",
|
||
31: "karty-lenorman-solnce",
|
||
32: "karty-lenorman-luna",
|
||
33: "karty-lenorman-klyuch",
|
||
34: "karty-lenorman-ryby",
|
||
35: "karty-lenorman-yakor",
|
||
36: "karty-lenorman-krest",
|
||
}
|
||
|
||
|
||
def slugify_url(url: str) -> str:
|
||
s = re.sub(r"^https?://", "", url)
|
||
s = re.sub(r"[^a-zA-Z0-9._-]+", "_", s)
|
||
return s[:180]
|
||
|
||
|
||
def fetch(url: str, force: bool = False) -> tuple[int, str, str]:
|
||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||
cache = HTML_DIR / (slugify_url(url) + ".html")
|
||
if cache.exists() and not force and cache.stat().st_size > 400:
|
||
return 200, url, cache.read_text(encoding="utf-8", errors="replace")
|
||
req = urllib.request.Request(
|
||
url,
|
||
headers={
|
||
"User-Agent": UA,
|
||
"Accept-Language": "ru,en;q=0.8",
|
||
"Accept": "text/html,application/xhtml+xml",
|
||
},
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
|
||
raw = resp.read()
|
||
enc = resp.headers.get_content_charset() or "utf-8"
|
||
text = raw.decode(enc, errors="replace")
|
||
final = str(resp.url)
|
||
status = resp.status
|
||
except urllib.error.HTTPError as e:
|
||
text = e.read().decode("utf-8", errors="replace") if e.fp else ""
|
||
cache.write_text(text, encoding="utf-8")
|
||
return e.code, url, text
|
||
except Exception as e:
|
||
return 0, url, f"ERROR {type(e).__name__}: {e}"
|
||
cache.write_text(text, encoding="utf-8")
|
||
time.sleep(SLEEP)
|
||
return status, final, text
|
||
|
||
|
||
def strip_tags(html: str) -> str:
|
||
html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
|
||
html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
|
||
html = re.sub(r"(?is)<noscript[^>]*>.*?</noscript>", " ", html)
|
||
html = re.sub(r"(?i)<br\s*/?>", "\n", html)
|
||
html = re.sub(r"(?i)</p>", "\n\n", html)
|
||
html = re.sub(r"(?i)</div>", "\n", html)
|
||
html = re.sub(r"(?i)</li>", "\n", html)
|
||
html = re.sub(r"(?i)</h[1-6]>", "\n", html)
|
||
html = re.sub(r"(?i)</tr>", "\n", html)
|
||
html = re.sub(r"(?i)</td>", " | ", html)
|
||
html = re.sub(r"(?i)</th>", " | ", html)
|
||
html = re.sub(r"<[^>]+>", " ", html)
|
||
html = htmlmod.unescape(html)
|
||
html = html.replace("\xa0", " ").replace(" ", " ")
|
||
html = re.sub(r"[ \t]+", " ", html)
|
||
html = re.sub(r"\n[ \t]+", "\n", html)
|
||
html = re.sub(r"\n{3,}", "\n\n", html)
|
||
return html.strip()
|
||
|
||
|
||
def magya_main(html: str) -> str:
|
||
m = re.search(r"(?is)<h1[^>]*>.*?</h1>(.*)", html)
|
||
body = m.group(1) if m else html
|
||
cut = re.search(
|
||
r"(?is)(Добавить комментарий|Судьба и случай рекомендуют|Рейтинг:)",
|
||
body,
|
||
)
|
||
if cut:
|
||
body = body[: cut.start()]
|
||
text = strip_tags(body)
|
||
# drop leftover css
|
||
text = re.sub(r"\.r_adapt_zag[\s\S]{0,400}?\}", "", text)
|
||
text = re.sub(r"blockquote,[\s\S]{0,500}?inside;\s*\}", "", text)
|
||
text = re.sub(r"li p \{[^}]+\}", "", text)
|
||
text = re.sub(r"Гадания\s*>\s*.*?Всадник.*?\n", "", text)
|
||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||
return text.strip()
|
||
|
||
|
||
def karusel_content(html: str) -> str:
|
||
m = re.search(r'(?is)<div[^>]+class="[^"]*entry-content[^"]*"[^>]*>(.*)', html)
|
||
body = m.group(1) if m else html
|
||
cut = re.search(r'(?is)<div[^>]+class="[^"]*(sharedaddy|yarpp|comments)', body)
|
||
if cut:
|
||
body = body[: cut.start()]
|
||
return strip_tags(body)
|
||
|
||
|
||
def resolve_name(token: str) -> int | None:
|
||
t = token.strip().lower()
|
||
t = t.replace("ё", "е")
|
||
t = re.sub(r"[^а-яa-z0-9]+", " ", t).strip()
|
||
if not t:
|
||
return None
|
||
if t in ALIAS_TO_NUM:
|
||
return ALIAS_TO_NUM[t]
|
||
# first word
|
||
w = t.split()[0]
|
||
return ALIAS_TO_NUM.get(w)
|
||
|
||
|
||
def pair_key(a: int, b: int) -> str:
|
||
return f"{min(a, b):02d}-{max(a, b):02d}"
|
||
|
||
|
||
def add_meaning(store, num: int, source: str, url: str, kind: str, text: str):
|
||
text = text.strip()
|
||
if len(text) < 40:
|
||
return
|
||
store["cards"][str(num)]["meanings"].append(
|
||
{"source": source, "kind": kind, "url": url, "text": text}
|
||
)
|
||
|
||
|
||
def add_pair(store, a: int, b: int, source: str, url: str, text: str, order: str):
|
||
if a == b:
|
||
return
|
||
text = re.sub(r"\s+", " ", text).strip(" -—–|")
|
||
if len(text) < 8:
|
||
return
|
||
key = pair_key(a, b)
|
||
bucket = store["pairs"].setdefault(
|
||
key,
|
||
{"a": min(a, b), "b": max(a, b), "readings": []},
|
||
)
|
||
# de-dupe identical source+text
|
||
for r in bucket["readings"]:
|
||
if r["source"] == source and r["text"] == text:
|
||
return
|
||
bucket["readings"].append(
|
||
{
|
||
"source": source,
|
||
"url": url,
|
||
"order": order, # a_then_b | undirected
|
||
"left": a,
|
||
"right": b,
|
||
"text": text,
|
||
}
|
||
)
|
||
|
||
|
||
def parse_karusel_combo_table(html: str, left: int, url: str, store) -> int:
|
||
n = 0
|
||
# rows: number | NAME | meaning
|
||
for m in re.finditer(
|
||
r"(?is)<tr[^>]*>\s*<td[^>]*>\s*(\d{1,2})\s*</td>\s*<td[^>]*>\s*([^<]+)</td>\s*<td[^>]*>(.*?)</td>",
|
||
html,
|
||
):
|
||
right = int(m.group(1))
|
||
meaning = strip_tags(m.group(3))
|
||
if right == left or right < 1 or right > 36:
|
||
continue
|
||
add_pair(store, left, right, "karusel_table", url, meaning, "a_then_b")
|
||
n += 1
|
||
if n:
|
||
return n
|
||
# markdown-like leftover or plain | 1 | ВСАДНИК | text |
|
||
text = strip_tags(html)
|
||
for m in re.finditer(
|
||
r"(?m)^\s*(\d{1,2})\s*\|\s*([А-ЯЁа-яёA-Za-z\s]+)\s*\|\s*(.+)$",
|
||
text,
|
||
):
|
||
right = int(m.group(1))
|
||
meaning = m.group(3).strip()
|
||
if right == left or right < 1 or right > 36:
|
||
continue
|
||
add_pair(store, left, right, "karusel_table", url, meaning, "a_then_b")
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def parse_karusel_short_pairs(html: str, url: str, store) -> int:
|
||
text = karusel_content(html)
|
||
n = 0
|
||
current_left = None
|
||
for line in text.splitlines():
|
||
line = line.strip()
|
||
m_head = re.match(
|
||
r"^(?:#{0,6}\s*)?(\d{1,2})\.\s*([А-ЯЁа-яёA-Za-z]+)\s*\+",
|
||
line,
|
||
)
|
||
if m_head:
|
||
current_left = int(m_head.group(1))
|
||
continue
|
||
m_pair = re.match(
|
||
r"^(\d{1,2})\s+([А-ЯЁа-яёA-Za-z\-]+)\s*[—–\-]\s*(.+)$",
|
||
line,
|
||
)
|
||
if m_pair and current_left:
|
||
right = int(m_pair.group(1))
|
||
add_pair(
|
||
store,
|
||
current_left,
|
||
right,
|
||
"karusel_short",
|
||
url,
|
||
m_pair.group(3),
|
||
"a_then_b",
|
||
)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def parse_karusel_triads(html: str, url: str, store) -> int:
|
||
text = karusel_content(html)
|
||
n = 0
|
||
for m in re.finditer(
|
||
r"(\d{1,2})\s*\+\s*(\d{1,2})\s*\+\s*(\d{1,2})\s*=\s*(.+)",
|
||
text,
|
||
):
|
||
a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||
if not all(1 <= x <= 36 for x in (a, b, c)):
|
||
continue
|
||
store["triads"].append(
|
||
{
|
||
"cards": [a, b, c],
|
||
"source": "karusel_triad",
|
||
"url": url,
|
||
"text": m.group(4).strip(),
|
||
}
|
||
)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def extract_divs_by_class(html: str, cls: str) -> list[str]:
|
||
"""Return inner HTML of each <div class="cls"> with nested divs balanced."""
|
||
pat = re.compile(rf'<div\s+class="{re.escape(cls)}">', re.I)
|
||
out: list[str] = []
|
||
i = 0
|
||
while True:
|
||
m = pat.search(html, i)
|
||
if not m:
|
||
break
|
||
pos = m.end()
|
||
depth = 1
|
||
j = pos
|
||
captured = None
|
||
while j < len(html) and depth:
|
||
nxt_open = html.find("<div", j)
|
||
nxt_close = html.find("</div>", j)
|
||
if nxt_close < 0:
|
||
break
|
||
if nxt_open != -1 and nxt_open < nxt_close:
|
||
depth += 1
|
||
j = nxt_open + 4
|
||
else:
|
||
depth -= 1
|
||
if depth == 0:
|
||
captured = html[pos:nxt_close]
|
||
i = nxt_close + 6
|
||
break
|
||
j = nxt_close + 6
|
||
if captured is None:
|
||
break
|
||
out.append(captured)
|
||
return out
|
||
|
||
|
||
def parse_ezo_thread(html: str, url: str, store) -> dict:
|
||
stats = {"meanings": 0, "pairs": 0, "extras": 0}
|
||
posts = extract_divs_by_class(html, "content")
|
||
extra_re = re.compile(
|
||
r"(?i)перевёрнут|карта дня|кратк\w+ значен|комбинац|магическ|кармическ|финансов|любви|работ"
|
||
)
|
||
for raw in posts:
|
||
text = strip_tags(raw)
|
||
current_left = None
|
||
head = re.search(
|
||
r"(?i)(?:^|\n)\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\s]{1,24})\s*(?:\(|$)",
|
||
text,
|
||
)
|
||
if not head:
|
||
head_html = re.search(
|
||
r'color:#008000["\']?[^>]*>\s*(?:(\d{1,2})\s+)?([А-ЯЁа-яёA-Za-z][^<(]{1,40})',
|
||
raw,
|
||
)
|
||
if head_html:
|
||
class _H:
|
||
def group(self, i):
|
||
return head_html.group(i)
|
||
head = _H()
|
||
if head:
|
||
num = int(head.group(1)) if head.group(1) else resolve_name(head.group(2))
|
||
if isinstance(num, int) and 1 <= num <= 36:
|
||
current_left = num
|
||
add_meaning(store, num, "ezoplaneta", url, "card", text)
|
||
stats["meanings"] += 1
|
||
elif extra_re.search(text[:400]) and len(text) > 200:
|
||
title = re.sub(r"\s+", " ", text.split("\n", 1)[0])[:90]
|
||
store["extras"].append(
|
||
{"source": "ezoplaneta", "url": url, "title": title, "text": text}
|
||
)
|
||
stats["extras"] += 1
|
||
|
||
if not current_left:
|
||
continue
|
||
# Early posts: <strong>КЛЕВЕР</strong>-Скорое...
|
||
# Later posts: <strong>Всадник</strong><br>- Неожиданная...
|
||
for m in re.finditer(
|
||
r"<strong[^>]*>\s*([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-\s]{2,24}?)\s*</strong>"
|
||
r"\s*(?:<br\s*/?>|\s)*[-—–:]?\s*([^<]+)",
|
||
raw,
|
||
):
|
||
right = resolve_name(m.group(1))
|
||
meaning = m.group(2).strip()
|
||
if not right or right == current_left:
|
||
continue
|
||
if len(meaning) < 4:
|
||
continue
|
||
add_pair(
|
||
store,
|
||
current_left,
|
||
right,
|
||
"ezoplaneta",
|
||
url,
|
||
meaning,
|
||
"a_then_b",
|
||
)
|
||
stats["pairs"] += 1
|
||
return stats
|
||
|
||
|
||
def parse_magya_meaning(html: str, num: int, url: str, store) -> int:
|
||
text = magya_main(html)
|
||
# split off combinations section
|
||
parts = re.split(r"(?i)Сочетание карты .+? с другими картами", text, maxsplit=1)
|
||
main = parts[0].strip()
|
||
add_meaning(store, num, "magya", url, "card", main)
|
||
n = 0
|
||
if len(parts) > 1:
|
||
combo_block = parts[1]
|
||
for m in re.finditer(
|
||
r"(?m)^([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-]{2,20})\s*[—–\-]\s*(.+)$",
|
||
combo_block,
|
||
):
|
||
right = resolve_name(m.group(1))
|
||
if not right or right == num:
|
||
continue
|
||
add_pair(store, num, right, "magya_meaning", url, m.group(2), "a_then_b")
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def parse_magya_pair(html: str, url: str, store) -> bool:
|
||
m = re.search(r"/sochetanie_kart_lenorman/([a-z]+)_i_([a-z]+)", url)
|
||
if not m:
|
||
return False
|
||
slug_a, slug_b = m.group(1), m.group(2)
|
||
inv = {v: k for k, v in MAGYA_SLUG.items()}
|
||
a = inv.get(slug_a)
|
||
b = inv.get(slug_b)
|
||
if not a or not b:
|
||
return False
|
||
text = magya_main(html)
|
||
text = re.sub(r"(?is)Смотрите также:[\s\S]+$", "", text).strip()
|
||
text = re.sub(r"(?is)Гадания\s*>[\s\S]{0,250}?\n+", "", text)
|
||
text = re.sub(
|
||
r"(?is)^(?:Значение карт Ленорман\s*>\s*)?(?:Сочетание карт Ленорман\s*>\s*)?[А-ЯЁа-яёA-Za-z\s]+и [А-ЯЁа-яёA-Za-z\s]+\s*",
|
||
"",
|
||
text,
|
||
count=1,
|
||
)
|
||
text = re.sub(r"\s+", " ", text).strip()
|
||
if len(text) < 40:
|
||
return False
|
||
add_pair(store, a, b, "magya_pair", url, text, "a_then_b")
|
||
return True
|
||
|
||
|
||
def empty_store() -> dict:
|
||
cards = {}
|
||
for n, cid, name, aliases in CARDS:
|
||
cards[str(n)] = {
|
||
"id": cid,
|
||
"number": n,
|
||
"name": name,
|
||
"aliases": aliases,
|
||
"meanings": [],
|
||
}
|
||
return {
|
||
"meta": {
|
||
"system": "lenormand",
|
||
"cards": 36,
|
||
"sources": {
|
||
"karusel": "https://karusel-magii.ru/",
|
||
"ezoplaneta": "https://ezoplaneta.ru/viewtopic.php?t=1913",
|
||
"magya": "https://magya-online.ru/",
|
||
},
|
||
"note": "Источники расходятся: readings[] хранит все версии без схлопывания.",
|
||
},
|
||
"cards": cards,
|
||
"pairs": {},
|
||
"triads": [],
|
||
"extras": [],
|
||
"errors": [],
|
||
}
|
||
|
||
|
||
def scrape(limit_pairs: int | None = None) -> dict:
|
||
store = empty_store()
|
||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
# ---------- KARUSEL ----------
|
||
print("== karusel meanings ==")
|
||
for n, slug in KARUSEL_MEAN.items():
|
||
url = f"https://karusel-magii.ru/{slug}/"
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
print(f" FAIL {n} {slug} {st}")
|
||
continue
|
||
text = karusel_content(html)
|
||
add_meaning(store, n, "karusel", final, "card", text)
|
||
print(f" mean {n:02d} {BY_NUM[n][1]} chars={len(text)}")
|
||
|
||
url = "https://karusel-magii.ru/znachenie-kart-lenorman/"
|
||
st, final, html = fetch(url)
|
||
if st == 200:
|
||
add_meaning(store, 1, "karusel", final, "hub_skipped", "") # noop
|
||
# keep as extra hub
|
||
store["extras"].append(
|
||
{
|
||
"source": "karusel",
|
||
"url": final,
|
||
"title": "Значение карт Ленорман (сводная)",
|
||
"text": karusel_content(html)[:20000],
|
||
}
|
||
)
|
||
|
||
print("== karusel combo tables ==")
|
||
for n, slug in KARUSEL_COMBO.items():
|
||
url = f"https://karusel-magii.ru/{slug}/"
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
print(f" FAIL combo {n} {st}")
|
||
continue
|
||
k = parse_karusel_combo_table(html, n, final, store)
|
||
print(f" table {n:02d} {BY_NUM[n][1]} pairs={k}")
|
||
|
||
print("== karusel short pairs + triads ==")
|
||
url = "https://karusel-magii.ru/znachenie-i-sochetanie-kart-lenorman/"
|
||
st, final, html = fetch(url)
|
||
if st == 200:
|
||
k = parse_karusel_short_pairs(html, final, store)
|
||
print(f" short pairs {k}")
|
||
url = "https://karusel-magii.ru/sochetanie-kart-lenorman/"
|
||
st, final, html = fetch(url)
|
||
if st == 200:
|
||
t = parse_karusel_triads(html, final, store)
|
||
print(f" triads {t}")
|
||
|
||
# ---------- EZOPLANETA ----------
|
||
print("== ezoplaneta thread ==")
|
||
for start in (0, 10, 20, 30, 40):
|
||
url = (
|
||
"https://ezoplaneta.ru/viewtopic.php?t=1913"
|
||
if start == 0
|
||
else f"https://ezoplaneta.ru/viewtopic.php?t=1913&start={start}"
|
||
)
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
print(f" FAIL start={start} {st}")
|
||
continue
|
||
stats = parse_ezo_thread(html, final, store)
|
||
print(f" page start={start} {stats}")
|
||
|
||
# ---------- MAGYA meanings ----------
|
||
print("== magya meanings ==")
|
||
pair_urls: list[str] = []
|
||
for n, slug in MAGYA_SLUG.items():
|
||
url = f"https://magya-online.ru/znachenie_kart_lenorman/{slug}"
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
print(f" FAIL mean {slug} {st}")
|
||
continue
|
||
k = parse_magya_meaning(html, n, final, store)
|
||
print(f" mean {n:02d} {slug} combos={k}")
|
||
|
||
hub = f"https://magya-online.ru/sochetanie_kart_lenorman/{slug}"
|
||
st2, final2, html2 = fetch(hub)
|
||
if st2 == 200:
|
||
found = re.findall(
|
||
rf"""href=['"](/sochetanie_kart_lenorman/{slug}_i_[a-z]+)['"]""",
|
||
html2,
|
||
)
|
||
for rel in found:
|
||
pair_urls.append("https://magya-online.ru" + rel)
|
||
|
||
# also generate directed slugs — magya uses cardA_i_cardB
|
||
for a, sa in MAGYA_SLUG.items():
|
||
for b, sb in MAGYA_SLUG.items():
|
||
if a == b:
|
||
continue
|
||
pair_urls.append(
|
||
f"https://magya-online.ru/sochetanie_kart_lenorman/{sa}_i_{sb}"
|
||
)
|
||
|
||
# unique preserve order
|
||
seen = set()
|
||
uniq = []
|
||
for u in pair_urls:
|
||
if u not in seen:
|
||
seen.add(u)
|
||
uniq.append(u)
|
||
if limit_pairs:
|
||
uniq = uniq[:limit_pairs]
|
||
print(f"== magya pair essays {len(uniq)} ==")
|
||
ok = 0
|
||
for i, url in enumerate(uniq, 1):
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
continue
|
||
if parse_magya_pair(html, final, store):
|
||
ok += 1
|
||
if i % 50 == 0:
|
||
print(f" pairs {i}/{len(uniq)} ok={ok}")
|
||
print(f" pair essays ok={ok}/{len(uniq)}")
|
||
|
||
return store
|
||
|
||
|
||
def coverage(store: dict) -> str:
|
||
lines = [
|
||
"# Lenormand corpus coverage",
|
||
"",
|
||
f"Карт: {len(store['cards'])}",
|
||
f"Пар с хотя бы одним чтением: {len(store['pairs'])} / 630 undirected",
|
||
f"Триад: {len(store['triads'])}",
|
||
f"Доп. разделов: {len(store['extras'])}",
|
||
f"Ошибок загрузки: {len(store['errors'])}",
|
||
"",
|
||
"## Значения по картам (число источников)",
|
||
"",
|
||
]
|
||
for n in range(1, 37):
|
||
c = store["cards"][str(n)]
|
||
srcs = sorted({m["source"] for m in c["meanings"] if m.get("text")})
|
||
lines.append(f"- {n:02d} {c['name']}: {len(c['meanings'])} записей ({', '.join(srcs) or '—'})")
|
||
|
||
by_src = defaultdict(int)
|
||
disagree = 0
|
||
for key, bucket in store["pairs"].items():
|
||
srcs = {r["source"] for r in bucket["readings"]}
|
||
for s in srcs:
|
||
by_src[s] += 1
|
||
if len(srcs) >= 2:
|
||
disagree += 1
|
||
lines += [
|
||
"",
|
||
"## Пары по источнику",
|
||
"",
|
||
]
|
||
for s, n in sorted(by_src.items()):
|
||
lines.append(f"- {s}: {n}")
|
||
lines += [
|
||
"",
|
||
f"Пар, где источники расходятся (2+ источника): {disagree}",
|
||
"",
|
||
]
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def main() -> None:
|
||
import argparse
|
||
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--limit-pairs", type=int, default=None)
|
||
ap.add_argument("--force", action="store_true")
|
||
args = ap.parse_args()
|
||
if args.force:
|
||
for p in HTML_DIR.glob("*.html"):
|
||
p.unlink()
|
||
store = scrape(limit_pairs=args.limit_pairs)
|
||
store["meta"]["pair_count"] = len(store["pairs"])
|
||
store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
|
||
CORPUS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||
CORPUS_PATH.write_text(
|
||
json.dumps(store, ensure_ascii=False, indent=2) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
cov = coverage(store)
|
||
COV_PATH.write_text(cov, encoding="utf-8")
|
||
print("\n" + cov)
|
||
print("wrote", CORPUS_PATH)
|
||
print("wrote", COV_PATH)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|