180 lines
6.7 KiB
Python
180 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Scrape Ogham few meanings from dedicated sites.
|
|
|
|
Sources:
|
|
- oghamlore.com (upright / reversed)
|
|
- druidogham.wordpress.com (well / ill dignified)
|
|
- globalspiritualstudies.com (chart)
|
|
|
|
Ogham has no Lenormand-style pair tables; combinations in a spread
|
|
are synthesized later from two fews' meanings.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import html as htmlmod
|
|
import json
|
|
import re
|
|
import ssl
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / "oracle_enricher" / "out" / "ogham_scrape" / "html"
|
|
CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "ogham_corpus.json"
|
|
CTX = ssl.create_default_context()
|
|
UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
|
|
SLEEP = 0.15
|
|
|
|
# canon_id, name_en, tree_ru, druidogham path
|
|
FEDA = [
|
|
("beith", "Beith", "берёза", "the-fews-air/beith"),
|
|
("luis", "Luis", "рябина", "the-fews-air/luis"),
|
|
("fern", "Fearn", "ольха", "the-fews-air/fearn"),
|
|
("sail", "Sail", "ива", "the-fews-air/saille"),
|
|
("nion", "Nion", "ясень", "the-fews-air/nuin"),
|
|
("uath", "Huath", "боярышник", "the-fews-fire/huath"),
|
|
("duir", "Duir", "дуб", "the-fews-fire/duir"),
|
|
("tinne", "Tinne", "остролист", "the-fews-fire/tinne"),
|
|
("coll", "Coll", "орешник", "the-fews-fire/coll"),
|
|
("ceirt", "Quert", "яблоня", "the-fews-fire/quiert"),
|
|
("muin", "Muin", "лоза", "the-fews-water/muin"),
|
|
("gort", "Gort", "плющ", "the-fews-water/gort"),
|
|
("ngetal", "nGetal", "тростник", "the-fews-water/ngetal"),
|
|
("straif", "Straif", "тёрн", "the-fews-water/straif"),
|
|
("ruis", "Ruis", "бузина", "the-fews-water/ruis"),
|
|
("ailm", "Ailm", "сосна", "the-fews-earth/ailm"),
|
|
("onn", "Onn", "дрок", "the-fews-earth/onn"),
|
|
("ur", "Ur", "вереск", "the-fews-earth/ur"),
|
|
("eadha", "Eadha", "осина", "the-fews-earth/eadha"),
|
|
("idho", "Idho", "тис", "the-fews-earth/ioho"),
|
|
]
|
|
|
|
|
|
def slugify(url: str) -> str:
|
|
s = re.sub(r"^https?://", "", url)
|
|
return re.sub(r"[^a-zA-Z0-9._-]+", "_", s)[:180]
|
|
|
|
|
|
def fetch(url: str) -> tuple[int, str]:
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
cache = OUT / (slugify(url) + ".html")
|
|
if cache.exists() and cache.stat().st_size > 400:
|
|
return 200, cache.read_text(encoding="utf-8", errors="replace")
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en"})
|
|
try:
|
|
with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
|
|
raw = resp.read()
|
|
enc = resp.headers.get_content_charset() or "utf-8"
|
|
text = raw.decode(enc, errors="replace")
|
|
status = resp.status
|
|
except urllib.error.HTTPError as e:
|
|
text = e.read().decode("utf-8", errors="replace") if e.fp else ""
|
|
cache.write_text(text, encoding="utf-8")
|
|
return e.code, text
|
|
except Exception as e:
|
|
return 0, str(e)
|
|
cache.write_text(text, encoding="utf-8")
|
|
time.sleep(SLEEP)
|
|
return status, text
|
|
|
|
|
|
def strip_tags(html: str) -> str:
|
|
html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
|
|
html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
|
|
html = re.sub(r"(?i)<br\s*/?>", "\n", html)
|
|
html = re.sub(r"(?i)</p>", "\n\n", html)
|
|
html = re.sub(r"(?i)</h[1-6]>", "\n", html)
|
|
html = re.sub(r"<[^>]+>", " ", html)
|
|
html = htmlmod.unescape(html).replace("\xa0", " ")
|
|
html = re.sub(r"[ \t]+", " ", html)
|
|
html = re.sub(r"\n{3,}", "\n\n", html)
|
|
return html.strip()
|
|
|
|
|
|
def parse_oghamlore(html: str) -> dict[str, str]:
|
|
"""Map English few name -> block of upright/reversed text."""
|
|
text = strip_tags(html)
|
|
out = {}
|
|
names = [en for _i, en, _t, _p in FEDA]
|
|
# split around "Beith (B) · Birch tree" style headers
|
|
parts = re.split(r"\n(?=(?:Beith|Luis|Fearn|Sail|Nion|Huath|Dair|Duir|Tinne|Coll|Quert|Muin|Gort|nGéadal|nGetal|Straif|Ruis|Ailm|Onn|Úr|Ur|Edad|Eadha|Idad|Idho)\b)", text)
|
|
for part in parts:
|
|
head = part.strip().split("\n", 1)[0][:40]
|
|
for en in names:
|
|
if head.lower().startswith(en.lower()) or f"{en} (" in head:
|
|
out[en] = part.strip()[:4000]
|
|
break
|
|
# Dair vs Duir
|
|
if head.lower().startswith("dair"):
|
|
out["Duir"] = part.strip()[:4000]
|
|
return out
|
|
|
|
|
|
def wp_content(html: str) -> str:
|
|
m = re.search(r'(?is)<div[^>]+class="[^"]*entry-content[^"]*"[^>]*>(.*)', html)
|
|
body = m.group(1) if m else html
|
|
for marker in ("sharedaddy", "jp-relatedposts", "comments"):
|
|
i = body.find(marker)
|
|
if i > 400:
|
|
body = body[:i]
|
|
break
|
|
return strip_tags(body)[:8000]
|
|
|
|
|
|
def main() -> None:
|
|
store = {
|
|
"meta": {
|
|
"system": "ogham",
|
|
"note": "Пары огама в источниках не таблицируются; сочетания — соседство fews в раскладе, сводятся из значений.",
|
|
},
|
|
"cards": {},
|
|
"errors": [],
|
|
}
|
|
for cid, en, tree, _p in FEDA:
|
|
store["cards"][cid] = {
|
|
"id": cid,
|
|
"name_en": en,
|
|
"tree_ru": tree,
|
|
"meanings": [],
|
|
}
|
|
|
|
print("== oghamlore ==")
|
|
st, html = fetch("https://oghamlore.com/learn/ogham-divination/")
|
|
lore = parse_oghamlore(html) if st == 200 else {}
|
|
print(" parsed", list(lore.keys()))
|
|
for cid, en, tree, _p in FEDA:
|
|
if en in lore:
|
|
store["cards"][cid]["meanings"].append(
|
|
{"source": "oghamlore", "url": "https://oghamlore.com/learn/ogham-divination/", "text": lore[en]}
|
|
)
|
|
|
|
print("== druidogham ==")
|
|
for cid, en, tree, path in FEDA:
|
|
url = f"https://druidogham.wordpress.com/{path}/"
|
|
st, html = fetch(url)
|
|
if st != 200:
|
|
store["errors"].append({"url": url, "status": st})
|
|
print(" FAIL", cid, st)
|
|
continue
|
|
text = wp_content(html)
|
|
store["cards"][cid]["meanings"].append({"source": "druidogham", "url": url, "text": text})
|
|
print(f" {cid:8} chars={len(text)}")
|
|
|
|
print("== globalspiritualstudies chart ==")
|
|
st, html = fetch("https://globalspiritualstudies.com/free-ogham-alphabet-chart/")
|
|
if st == 200:
|
|
text = strip_tags(html)
|
|
store["meta"]["chart"] = text[:12000]
|
|
print(" chart chars", len(text))
|
|
|
|
nmean = sum(1 for c in store["cards"].values() if c["meanings"])
|
|
CORPUS.parent.mkdir(parents=True, exist_ok=True)
|
|
CORPUS.write_text(json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("wrote", CORPUS, "fews_with_meanings", nmean, "errors", len(store["errors"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|