feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets
This commit is contained in:
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scrape Major Arcana meanings + major-major pairs from magya, karusel, ezoplaneta.
|
||||
|
||||
Writes oracle_enricher/data/sources/tarot_majors_corpus.json
|
||||
HTML cache: oracle_enricher/out/tarot_scrape/html/
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html as htmlmod
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT_DIR = ROOT / "oracle_enricher" / "out" / "tarot_scrape"
|
||||
HTML_DIR = OUT_DIR / "html"
|
||||
CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "tarot_majors_corpus.json"
|
||||
|
||||
CTX = ssl.create_default_context()
|
||||
UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
|
||||
SLEEP = 0.12
|
||||
|
||||
# n, id, name, magya slug, karusel slug, aliases
|
||||
CARDS = [
|
||||
(0, "fool", "Шут", "durak", "durak", ["шут", "дурак", "безумец"]),
|
||||
(1, "magician", "Маг", "mag", "mag", ["маг"]),
|
||||
(2, "high_priestess", "Верховная Жрица", "jrica", "verxovnaya-zhrica", ["жрица", "верховная жрица", "жрица"]),
|
||||
(3, "empress", "Императрица", "imperatrica", "imperatrica", ["императрица"]),
|
||||
(4, "emperor", "Император", "imperator", "imperator", ["император"]),
|
||||
(5, "hierophant", "Иерофант", "jrec", "ierofant", ["иерофант", "жрец", "папа"]),
|
||||
(6, "lovers", "Влюбленные", "vlublennie", "vlyublennye", ["влюбленные", "влюблённые"]),
|
||||
(7, "chariot", "Колесница", "kolesnica", "kolesnica", ["колесница"]),
|
||||
(8, "strength", "Сила", "sila", "sila", ["сила"]),
|
||||
(9, "hermit", "Отшельник", "otshelnik", "otshelnik", ["отшельник"]),
|
||||
(10, "wheel", "Колесо Фортуны", "koleso_fortuni", "koleso-fortuny", ["колесо фортуны", "колесо"]),
|
||||
(11, "justice", "Справедливость", "spravedlivost", "spravedlivost", ["справедливость", "правосудие"]),
|
||||
(12, "hanged_man", "Повешенный", "poveshenniy", "poveshennyj", ["повешенный"]),
|
||||
(13, "death", "Смерть", "smert", "smert", ["смерть"]),
|
||||
(14, "temperance", "Умеренность", "umerennost", "umerennost", ["умеренность"]),
|
||||
(15, "devil", "Дьявол", "dyavol", "dyavol", ["дьявол"]),
|
||||
(16, "tower", "Башня", "bashnya", "bashnya", ["башня"]),
|
||||
(17, "star", "Звезда", "zvezda", "zvezda", ["звезда"]),
|
||||
(18, "moon", "Луна", "luna", "luna", ["луна"]),
|
||||
(19, "sun", "Солнце", "solnce", "solnce", ["солнце"]),
|
||||
(20, "judgement", "Суд", "sud", "sud", ["суд"]),
|
||||
(21, "world", "Мир", "mir", "mir", ["мир"]),
|
||||
]
|
||||
BY_N = {rec[0]: rec for rec in CARDS}
|
||||
MAGYA = {n: slug for n, _i, _name, slug, _k, _a in CARDS}
|
||||
ALIAS = {}
|
||||
for n, _i, name, _m, _k, aliases in CARDS:
|
||||
ALIAS[name.lower()] = n
|
||||
for a in aliases:
|
||||
ALIAS[a] = n
|
||||
|
||||
|
||||
def slugify_url(url: str) -> str:
|
||||
s = re.sub(r"^https?://", "", url)
|
||||
return re.sub(r"[^a-zA-Z0-9._-]+", "_", s)[:180]
|
||||
|
||||
|
||||
def fetch(url: str) -> tuple[int, str, str]:
|
||||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache = HTML_DIR / (slugify_url(url) + ".html")
|
||||
if cache.exists() and cache.stat().st_size > 400:
|
||||
return 200, url, cache.read_text(encoding="utf-8", errors="replace")
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "ru"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
|
||||
raw = resp.read()
|
||||
enc = resp.headers.get_content_charset() or "utf-8"
|
||||
text = raw.decode(enc, errors="replace")
|
||||
status = resp.status
|
||||
final = str(resp.url)
|
||||
except urllib.error.HTTPError as e:
|
||||
text = e.read().decode("utf-8", errors="replace") if e.fp else ""
|
||||
cache.write_text(text, encoding="utf-8")
|
||||
return e.code, url, text
|
||||
except Exception as e:
|
||||
return 0, url, f"ERROR {type(e).__name__}: {e}"
|
||||
cache.write_text(text, encoding="utf-8")
|
||||
time.sleep(SLEEP)
|
||||
return status, final, text
|
||||
|
||||
|
||||
def strip_tags(html: str) -> str:
|
||||
html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
|
||||
html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
|
||||
html = re.sub(r"(?i)<br\s*/?>", "\n", html)
|
||||
html = re.sub(r"(?i)</p>", "\n\n", html)
|
||||
html = re.sub(r"(?i)</h[1-6]>", "\n", html)
|
||||
html = re.sub(r"(?i)</div>", "\n", html)
|
||||
html = re.sub(r"<[^>]+>", " ", html)
|
||||
html = htmlmod.unescape(html).replace("\xa0", " ")
|
||||
html = re.sub(r"[ \t]+", " ", html)
|
||||
html = re.sub(r"\n{3,}", "\n\n", html)
|
||||
return html.strip()
|
||||
|
||||
|
||||
def magya_main(html: str) -> str:
|
||||
m = re.search(r"(?is)<h1[^>]*>.*?</h1>(.*)", html)
|
||||
body = m.group(1) if m else html
|
||||
cut = re.search(r"(?is)(Добавить комментарий|Судьба и случай рекомендуют|Рейтинг:)", body)
|
||||
if cut:
|
||||
body = body[: cut.start()]
|
||||
text = strip_tags(body)
|
||||
text = re.sub(r"\.natal-ad-container[\s\S]{0,400}?\}", "", text)
|
||||
text = re.sub(r"@media[\s\S]{0,200}?\}", "", text)
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
|
||||
|
||||
def karusel_content(html: str) -> str:
|
||||
m = re.search(r'(?is)<div class="content clearfix">(.*)', html)
|
||||
body = m.group(1) if m else html
|
||||
for marker in ('id="sidebar', "sharedaddy", "yarpp"):
|
||||
i = body.find(marker)
|
||||
if i > 300:
|
||||
body = body[:i]
|
||||
break
|
||||
text = strip_tags(body)
|
||||
text = re.split(r"(?i)Смотрите также|Рекомендуем по этой теме", text)[0]
|
||||
return text.strip()
|
||||
|
||||
|
||||
def empty_store() -> dict:
|
||||
cards = {}
|
||||
for n, cid, name, _m, _k, aliases in CARDS:
|
||||
cards[str(n)] = {
|
||||
"id": cid,
|
||||
"number": n,
|
||||
"name": name,
|
||||
"aliases": aliases,
|
||||
"meanings": [],
|
||||
}
|
||||
return {"meta": {"system": "tarot_majors", "cards": 22}, "cards": cards, "pairs": {}, "errors": []}
|
||||
|
||||
|
||||
def add_meaning(store, n, source, url, text):
|
||||
text = text.strip()
|
||||
if len(text) < 40:
|
||||
return
|
||||
store["cards"][str(n)]["meanings"].append({"source": source, "url": url, "text": text})
|
||||
|
||||
|
||||
def add_pair(store, a, b, source, url, text):
|
||||
if a == b or a < 0 or b < 0:
|
||||
return
|
||||
text = re.sub(r"\s+", " ", text).strip(" -—")
|
||||
if len(text) < 20:
|
||||
return
|
||||
key = f"{min(a,b):02d}-{max(a,b):02d}"
|
||||
bucket = store["pairs"].setdefault(key, {"a": min(a, b), "b": max(a, b), "readings": []})
|
||||
for r in bucket["readings"]:
|
||||
if r["source"] == source and r["text"] == text:
|
||||
return
|
||||
bucket["readings"].append({"source": source, "url": url, "left": a, "right": b, "text": text})
|
||||
|
||||
|
||||
def scrape() -> dict:
|
||||
store = empty_store()
|
||||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print("== magya meanings ==")
|
||||
for n, _i, name, slug, _k, _a in CARDS:
|
||||
url = f"https://magya-online.ru/znachenie_kart_taro_raydera_weyta/{slug}"
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
print(f" FAIL mean {slug} {st}")
|
||||
continue
|
||||
text = magya_main(html)
|
||||
add_meaning(store, n, "magya", final, text)
|
||||
print(f" mean {n:02d} {name} chars={len(text)}")
|
||||
|
||||
print("== karusel meanings ==")
|
||||
for n, _i, name, _m, kslug, _a in CARDS:
|
||||
url = f"https://karusel-magii.ru/znachenie-kart-taro-pri-gadanii-tolkovanie-karty-taro-{kslug}/"
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
print(f" FAIL karusel {kslug} {st}")
|
||||
continue
|
||||
text = karusel_content(html)
|
||||
add_meaning(store, n, "karusel", final, text)
|
||||
print(f" karusel {n:02d} {name} chars={len(text)}")
|
||||
|
||||
print("== ezoplaneta love majors t=109 ==")
|
||||
st, final, html = fetch("https://ezoplaneta.ru/viewtopic.php?t=109")
|
||||
if st == 200:
|
||||
# split by ALL-CAPS names
|
||||
text = strip_tags(html)
|
||||
parts = re.split(r"\n(?=(?:ШУТ|МАГ|ВЕРХОВНАЯ ЖРИЦА|ИМПЕРАТРИЦА|ИМПЕРАТОР|ИЕРОФАНТ|ЖРЕЦ|ВЛЮБЛЕННЫЕ|КОЛЕСНИЦА|СИЛА|ОТШЕЛЬНИК|КОЛЕСО ФОРТУНЫ|СПРАВЕДЛИВОСТЬ|ПОВЕШЕННЫЙ|СМЕРТЬ|УМЕРЕННОСТЬ|ДЬЯВОЛ|БАШНЯ|ЗВЕЗДА|ЛУНА|СОЛНЦЕ|СУД|МИР)\b)", text)
|
||||
n_ezo = 0
|
||||
for part in parts:
|
||||
first = part.strip().split("\n", 1)[0][:40].lower()
|
||||
num = ALIAS.get(first.strip(" ."))
|
||||
if num is None:
|
||||
for alias, an in ALIAS.items():
|
||||
if first.startswith(alias):
|
||||
num = an
|
||||
break
|
||||
if num is None:
|
||||
continue
|
||||
add_meaning(store, num, "ezoplaneta", final, part.strip())
|
||||
n_ezo += 1
|
||||
print(f" ezo cards {n_ezo}")
|
||||
else:
|
||||
store["errors"].append({"url": "ezo t=109", "status": st})
|
||||
|
||||
print("== magya major-major pair essays ==")
|
||||
ok = 0
|
||||
total = 0
|
||||
for a, _ia, _na, sa, _ka, _aa in CARDS:
|
||||
for b, _ib, _nb, sb, _kb, _ab in CARDS:
|
||||
if a == b:
|
||||
continue
|
||||
total += 1
|
||||
url = f"https://magya-online.ru/sochetanie_kart_taro/{sa}_i_{sb}"
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
store["errors"].append({"url": url, "status": st})
|
||||
continue
|
||||
text = magya_main(html)
|
||||
if "выберите сочетание" in text.lower()[:400]:
|
||||
continue
|
||||
if len(text) < 80:
|
||||
continue
|
||||
add_pair(store, a, b, "magya_pair", final, text)
|
||||
ok += 1
|
||||
print(f" from {a:02d} pairs ok so far {ok}")
|
||||
print(f" pair essays kept={ok}/{total}")
|
||||
|
||||
# short combos from magya hubs
|
||||
print("== magya hub short combos ==")
|
||||
nshort = 0
|
||||
for n, _i, name, slug, _k, _a in CARDS:
|
||||
url = f"https://magya-online.ru/sochetanie_kart_taro/{slug}"
|
||||
st, final, html = fetch(url)
|
||||
if st != 200:
|
||||
continue
|
||||
text = magya_main(html)
|
||||
for m in re.finditer(
|
||||
r"(?m)^([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-\s]{2,40}?)\s*[—–\-]\s*(.+)$",
|
||||
text,
|
||||
):
|
||||
right = ALIAS.get(m.group(1).strip().lower())
|
||||
if right is None:
|
||||
right = ALIAS.get(m.group(1).strip().lower().split()[0])
|
||||
if right is None or right == n:
|
||||
continue
|
||||
add_pair(store, n, right, "magya_meaning", final, m.group(2))
|
||||
nshort += 1
|
||||
print(f" short combos {nshort}")
|
||||
return store
|
||||
|
||||
|
||||
def main() -> None:
|
||||
store = scrape()
|
||||
store["meta"]["pair_count"] = len(store["pairs"])
|
||||
store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
|
||||
CORPUS.parent.mkdir(parents=True, exist_ok=True)
|
||||
CORPUS.write_text(json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print("wrote", CORPUS)
|
||||
print("cards with meanings", sum(1 for c in store["cards"].values() if c["meanings"]))
|
||||
print("pairs", store["meta"]["pair_count"], "readings", store["meta"]["reading_count"])
|
||||
print("errors", len(store["errors"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user