275 lines
11 KiB
Python
275 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Scrape Major Arcana meanings + major-major pairs from magya, karusel, ezoplaneta.
|
||
|
||
Writes oracle_enricher/data/sources/tarot_majors_corpus.json
|
||
HTML cache: oracle_enricher/out/tarot_scrape/html/
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import html as htmlmod
|
||
import json
|
||
import re
|
||
import ssl
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OUT_DIR = ROOT / "oracle_enricher" / "out" / "tarot_scrape"
|
||
HTML_DIR = OUT_DIR / "html"
|
||
CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "tarot_majors_corpus.json"
|
||
|
||
CTX = ssl.create_default_context()
|
||
UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
|
||
SLEEP = 0.12
|
||
|
||
# n, id, name, magya slug, karusel slug, aliases
|
||
CARDS = [
|
||
(0, "fool", "Шут", "durak", "durak", ["шут", "дурак", "безумец"]),
|
||
(1, "magician", "Маг", "mag", "mag", ["маг"]),
|
||
(2, "high_priestess", "Верховная Жрица", "jrica", "verxovnaya-zhrica", ["жрица", "верховная жрица", "жрица"]),
|
||
(3, "empress", "Императрица", "imperatrica", "imperatrica", ["императрица"]),
|
||
(4, "emperor", "Император", "imperator", "imperator", ["император"]),
|
||
(5, "hierophant", "Иерофант", "jrec", "ierofant", ["иерофант", "жрец", "папа"]),
|
||
(6, "lovers", "Влюбленные", "vlublennie", "vlyublennye", ["влюбленные", "влюблённые"]),
|
||
(7, "chariot", "Колесница", "kolesnica", "kolesnica", ["колесница"]),
|
||
(8, "strength", "Сила", "sila", "sila", ["сила"]),
|
||
(9, "hermit", "Отшельник", "otshelnik", "otshelnik", ["отшельник"]),
|
||
(10, "wheel", "Колесо Фортуны", "koleso_fortuni", "koleso-fortuny", ["колесо фортуны", "колесо"]),
|
||
(11, "justice", "Справедливость", "spravedlivost", "spravedlivost", ["справедливость", "правосудие"]),
|
||
(12, "hanged_man", "Повешенный", "poveshenniy", "poveshennyj", ["повешенный"]),
|
||
(13, "death", "Смерть", "smert", "smert", ["смерть"]),
|
||
(14, "temperance", "Умеренность", "umerennost", "umerennost", ["умеренность"]),
|
||
(15, "devil", "Дьявол", "dyavol", "dyavol", ["дьявол"]),
|
||
(16, "tower", "Башня", "bashnya", "bashnya", ["башня"]),
|
||
(17, "star", "Звезда", "zvezda", "zvezda", ["звезда"]),
|
||
(18, "moon", "Луна", "luna", "luna", ["луна"]),
|
||
(19, "sun", "Солнце", "solnce", "solnce", ["солнце"]),
|
||
(20, "judgement", "Суд", "sud", "sud", ["суд"]),
|
||
(21, "world", "Мир", "mir", "mir", ["мир"]),
|
||
]
|
||
BY_N = {rec[0]: rec for rec in CARDS}
|
||
MAGYA = {n: slug for n, _i, _name, slug, _k, _a in CARDS}
|
||
ALIAS = {}
|
||
for n, _i, name, _m, _k, aliases in CARDS:
|
||
ALIAS[name.lower()] = n
|
||
for a in aliases:
|
||
ALIAS[a] = n
|
||
|
||
|
||
def slugify_url(url: str) -> str:
|
||
s = re.sub(r"^https?://", "", url)
|
||
return re.sub(r"[^a-zA-Z0-9._-]+", "_", s)[:180]
|
||
|
||
|
||
def fetch(url: str) -> tuple[int, str, str]:
|
||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||
cache = HTML_DIR / (slugify_url(url) + ".html")
|
||
if cache.exists() and cache.stat().st_size > 400:
|
||
return 200, url, cache.read_text(encoding="utf-8", errors="replace")
|
||
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "ru"})
|
||
try:
|
||
with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
|
||
raw = resp.read()
|
||
enc = resp.headers.get_content_charset() or "utf-8"
|
||
text = raw.decode(enc, errors="replace")
|
||
status = resp.status
|
||
final = str(resp.url)
|
||
except urllib.error.HTTPError as e:
|
||
text = e.read().decode("utf-8", errors="replace") if e.fp else ""
|
||
cache.write_text(text, encoding="utf-8")
|
||
return e.code, url, text
|
||
except Exception as e:
|
||
return 0, url, f"ERROR {type(e).__name__}: {e}"
|
||
cache.write_text(text, encoding="utf-8")
|
||
time.sleep(SLEEP)
|
||
return status, final, text
|
||
|
||
|
||
def strip_tags(html: str) -> str:
|
||
html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
|
||
html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
|
||
html = re.sub(r"(?i)<br\s*/?>", "\n", html)
|
||
html = re.sub(r"(?i)</p>", "\n\n", html)
|
||
html = re.sub(r"(?i)</h[1-6]>", "\n", html)
|
||
html = re.sub(r"(?i)</div>", "\n", html)
|
||
html = re.sub(r"<[^>]+>", " ", html)
|
||
html = htmlmod.unescape(html).replace("\xa0", " ")
|
||
html = re.sub(r"[ \t]+", " ", html)
|
||
html = re.sub(r"\n{3,}", "\n\n", html)
|
||
return html.strip()
|
||
|
||
|
||
def magya_main(html: str) -> str:
|
||
m = re.search(r"(?is)<h1[^>]*>.*?</h1>(.*)", html)
|
||
body = m.group(1) if m else html
|
||
cut = re.search(r"(?is)(Добавить комментарий|Судьба и случай рекомендуют|Рейтинг:)", body)
|
||
if cut:
|
||
body = body[: cut.start()]
|
||
text = strip_tags(body)
|
||
text = re.sub(r"\.natal-ad-container[\s\S]{0,400}?\}", "", text)
|
||
text = re.sub(r"@media[\s\S]{0,200}?\}", "", text)
|
||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||
|
||
|
||
def karusel_content(html: str) -> str:
|
||
m = re.search(r'(?is)<div class="content clearfix">(.*)', html)
|
||
body = m.group(1) if m else html
|
||
for marker in ('id="sidebar', "sharedaddy", "yarpp"):
|
||
i = body.find(marker)
|
||
if i > 300:
|
||
body = body[:i]
|
||
break
|
||
text = strip_tags(body)
|
||
text = re.split(r"(?i)Смотрите также|Рекомендуем по этой теме", text)[0]
|
||
return text.strip()
|
||
|
||
|
||
def empty_store() -> dict:
|
||
cards = {}
|
||
for n, cid, name, _m, _k, aliases in CARDS:
|
||
cards[str(n)] = {
|
||
"id": cid,
|
||
"number": n,
|
||
"name": name,
|
||
"aliases": aliases,
|
||
"meanings": [],
|
||
}
|
||
return {"meta": {"system": "tarot_majors", "cards": 22}, "cards": cards, "pairs": {}, "errors": []}
|
||
|
||
|
||
def add_meaning(store, n, source, url, text):
|
||
text = text.strip()
|
||
if len(text) < 40:
|
||
return
|
||
store["cards"][str(n)]["meanings"].append({"source": source, "url": url, "text": text})
|
||
|
||
|
||
def add_pair(store, a, b, source, url, text):
|
||
if a == b or a < 0 or b < 0:
|
||
return
|
||
text = re.sub(r"\s+", " ", text).strip(" -—")
|
||
if len(text) < 20:
|
||
return
|
||
key = f"{min(a,b):02d}-{max(a,b):02d}"
|
||
bucket = store["pairs"].setdefault(key, {"a": min(a, b), "b": max(a, b), "readings": []})
|
||
for r in bucket["readings"]:
|
||
if r["source"] == source and r["text"] == text:
|
||
return
|
||
bucket["readings"].append({"source": source, "url": url, "left": a, "right": b, "text": text})
|
||
|
||
|
||
def scrape() -> dict:
|
||
store = empty_store()
|
||
HTML_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
print("== magya meanings ==")
|
||
for n, _i, name, slug, _k, _a in CARDS:
|
||
url = f"https://magya-online.ru/znachenie_kart_taro_raydera_weyta/{slug}"
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
print(f" FAIL mean {slug} {st}")
|
||
continue
|
||
text = magya_main(html)
|
||
add_meaning(store, n, "magya", final, text)
|
||
print(f" mean {n:02d} {name} chars={len(text)}")
|
||
|
||
print("== karusel meanings ==")
|
||
for n, _i, name, _m, kslug, _a in CARDS:
|
||
url = f"https://karusel-magii.ru/znachenie-kart-taro-pri-gadanii-tolkovanie-karty-taro-{kslug}/"
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
print(f" FAIL karusel {kslug} {st}")
|
||
continue
|
||
text = karusel_content(html)
|
||
add_meaning(store, n, "karusel", final, text)
|
||
print(f" karusel {n:02d} {name} chars={len(text)}")
|
||
|
||
print("== ezoplaneta love majors t=109 ==")
|
||
st, final, html = fetch("https://ezoplaneta.ru/viewtopic.php?t=109")
|
||
if st == 200:
|
||
# split by ALL-CAPS names
|
||
text = strip_tags(html)
|
||
parts = re.split(r"\n(?=(?:ШУТ|МАГ|ВЕРХОВНАЯ ЖРИЦА|ИМПЕРАТРИЦА|ИМПЕРАТОР|ИЕРОФАНТ|ЖРЕЦ|ВЛЮБЛЕННЫЕ|КОЛЕСНИЦА|СИЛА|ОТШЕЛЬНИК|КОЛЕСО ФОРТУНЫ|СПРАВЕДЛИВОСТЬ|ПОВЕШЕННЫЙ|СМЕРТЬ|УМЕРЕННОСТЬ|ДЬЯВОЛ|БАШНЯ|ЗВЕЗДА|ЛУНА|СОЛНЦЕ|СУД|МИР)\b)", text)
|
||
n_ezo = 0
|
||
for part in parts:
|
||
first = part.strip().split("\n", 1)[0][:40].lower()
|
||
num = ALIAS.get(first.strip(" ."))
|
||
if num is None:
|
||
for alias, an in ALIAS.items():
|
||
if first.startswith(alias):
|
||
num = an
|
||
break
|
||
if num is None:
|
||
continue
|
||
add_meaning(store, num, "ezoplaneta", final, part.strip())
|
||
n_ezo += 1
|
||
print(f" ezo cards {n_ezo}")
|
||
else:
|
||
store["errors"].append({"url": "ezo t=109", "status": st})
|
||
|
||
print("== magya major-major pair essays ==")
|
||
ok = 0
|
||
total = 0
|
||
for a, _ia, _na, sa, _ka, _aa in CARDS:
|
||
for b, _ib, _nb, sb, _kb, _ab in CARDS:
|
||
if a == b:
|
||
continue
|
||
total += 1
|
||
url = f"https://magya-online.ru/sochetanie_kart_taro/{sa}_i_{sb}"
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
store["errors"].append({"url": url, "status": st})
|
||
continue
|
||
text = magya_main(html)
|
||
if "выберите сочетание" in text.lower()[:400]:
|
||
continue
|
||
if len(text) < 80:
|
||
continue
|
||
add_pair(store, a, b, "magya_pair", final, text)
|
||
ok += 1
|
||
print(f" from {a:02d} pairs ok so far {ok}")
|
||
print(f" pair essays kept={ok}/{total}")
|
||
|
||
# short combos from magya hubs
|
||
print("== magya hub short combos ==")
|
||
nshort = 0
|
||
for n, _i, name, slug, _k, _a in CARDS:
|
||
url = f"https://magya-online.ru/sochetanie_kart_taro/{slug}"
|
||
st, final, html = fetch(url)
|
||
if st != 200:
|
||
continue
|
||
text = magya_main(html)
|
||
for m in re.finditer(
|
||
r"(?m)^([А-ЯЁа-яёA-Za-z][А-ЯЁа-яёA-Za-z\-\s]{2,40}?)\s*[—–\-]\s*(.+)$",
|
||
text,
|
||
):
|
||
right = ALIAS.get(m.group(1).strip().lower())
|
||
if right is None:
|
||
right = ALIAS.get(m.group(1).strip().lower().split()[0])
|
||
if right is None or right == n:
|
||
continue
|
||
add_pair(store, n, right, "magya_meaning", final, m.group(2))
|
||
nshort += 1
|
||
print(f" short combos {nshort}")
|
||
return store
|
||
|
||
|
||
def main() -> None:
|
||
store = scrape()
|
||
store["meta"]["pair_count"] = len(store["pairs"])
|
||
store["meta"]["reading_count"] = sum(len(p["readings"]) for p in store["pairs"].values())
|
||
CORPUS.parent.mkdir(parents=True, exist_ok=True)
|
||
CORPUS.write_text(json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print("wrote", CORPUS)
|
||
print("cards with meanings", sum(1 for c in store["cards"].values() if c["meanings"]))
|
||
print("pairs", store["meta"]["pair_count"], "readings", store["meta"]["reading_count"])
|
||
print("errors", len(store["errors"]))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|