feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets

2026-10-09 15:26:58 +05:00
parent 7f91366201
commit 0b12e4583b
244 changed files with 611461 additions and 1617 deletions
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""Scrape Ogham few meanings from dedicated sites.
+
+Sources:
+  - oghamlore.com (upright / reversed)
+  - druidogham.wordpress.com (well / ill dignified)
+  - globalspiritualstudies.com (chart)
+
+Ogham has no Lenormand-style pair tables; combinations in a spread
+are synthesized later from two fews' meanings.
+"""
+from __future__ import annotations
+
+import html as htmlmod
+import json
+import re
+import ssl
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / "oracle_enricher" / "out" / "ogham_scrape" / "html"
+CORPUS = ROOT / "oracle_enricher" / "data" / "sources" / "ogham_corpus.json"
+CTX = ssl.create_default_context()
+UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36"
+SLEEP = 0.15
+
+# canon_id, name_en, tree_ru, druidogham path
+FEDA = [
+    ("beith", "Beith", "берёза", "the-fews-air/beith"),
+    ("luis", "Luis", "рябина", "the-fews-air/luis"),
+    ("fern", "Fearn", "ольха", "the-fews-air/fearn"),
+    ("sail", "Sail", "ива", "the-fews-air/saille"),
+    ("nion", "Nion", "ясень", "the-fews-air/nuin"),
+    ("uath", "Huath", "боярышник", "the-fews-fire/huath"),
+    ("duir", "Duir", "дуб", "the-fews-fire/duir"),
+    ("tinne", "Tinne", "остролист", "the-fews-fire/tinne"),
+    ("coll", "Coll", "орешник", "the-fews-fire/coll"),
+    ("ceirt", "Quert", "яблоня", "the-fews-fire/quiert"),
+    ("muin", "Muin", "лоза", "the-fews-water/muin"),
+    ("gort", "Gort", "плющ", "the-fews-water/gort"),
+    ("ngetal", "nGetal", "тростник", "the-fews-water/ngetal"),
+    ("straif", "Straif", "тёрн", "the-fews-water/straif"),
+    ("ruis", "Ruis", "бузина", "the-fews-water/ruis"),
+    ("ailm", "Ailm", "сосна", "the-fews-earth/ailm"),
+    ("onn", "Onn", "дрок", "the-fews-earth/onn"),
+    ("ur", "Ur", "вереск", "the-fews-earth/ur"),
+    ("eadha", "Eadha", "осина", "the-fews-earth/eadha"),
+    ("idho", "Idho", "тис", "the-fews-earth/ioho"),
+]
+
+
+def slugify(url: str) -> str:
+    s = re.sub(r"^https?://", "", url)
+    return re.sub(r"[^a-zA-Z0-9._-]+", "_", s)[:180]
+
+
+def fetch(url: str) -> tuple[int, str]:
+    OUT.mkdir(parents=True, exist_ok=True)
+    cache = OUT / (slugify(url) + ".html")
+    if cache.exists() and cache.stat().st_size > 400:
+        return 200, cache.read_text(encoding="utf-8", errors="replace")
+    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en"})
+    try:
+        with urllib.request.urlopen(req, context=CTX, timeout=40) as resp:
+            raw = resp.read()
+            enc = resp.headers.get_content_charset() or "utf-8"
+            text = raw.decode(enc, errors="replace")
+            status = resp.status
+    except urllib.error.HTTPError as e:
+        text = e.read().decode("utf-8", errors="replace") if e.fp else ""
+        cache.write_text(text, encoding="utf-8")
+        return e.code, text
+    except Exception as e:
+        return 0, str(e)
+    cache.write_text(text, encoding="utf-8")
+    time.sleep(SLEEP)
+    return status, text
+
+
+def strip_tags(html: str) -> str:
+    html = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", html)
+    html = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", html)
+    html = re.sub(r"(?i)<br\s*/?>", "\n", html)
+    html = re.sub(r"(?i)</p>", "\n\n", html)
+    html = re.sub(r"(?i)</h[1-6]>", "\n", html)
+    html = re.sub(r"<[^>]+>", " ", html)
+    html = htmlmod.unescape(html).replace("\xa0", " ")
+    html = re.sub(r"[ \t]+", " ", html)
+    html = re.sub(r"\n{3,}", "\n\n", html)
+    return html.strip()
+
+
+def parse_oghamlore(html: str) -> dict[str, str]:
+    """Map English few name -> block of upright/reversed text."""
+    text = strip_tags(html)
+    out = {}
+    names = [en for _i, en, _t, _p in FEDA]
+    # split around "Beith (B) · Birch tree" style headers
+    parts = re.split(r"\n(?=(?:Beith|Luis|Fearn|Sail|Nion|Huath|Dair|Duir|Tinne|Coll|Quert|Muin|Gort|nGéadal|nGetal|Straif|Ruis|Ailm|Onn|Úr|Ur|Edad|Eadha|Idad|Idho)\b)", text)
+    for part in parts:
+        head = part.strip().split("\n", 1)[0][:40]
+        for en in names:
+            if head.lower().startswith(en.lower()) or f"{en} (" in head:
+                out[en] = part.strip()[:4000]
+                break
+        # Dair vs Duir
+        if head.lower().startswith("dair"):
+            out["Duir"] = part.strip()[:4000]
+    return out
+
+
+def wp_content(html: str) -> str:
+    m = re.search(r'(?is)<div[^>]+class="[^"]*entry-content[^"]*"[^>]*>(.*)', html)
+    body = m.group(1) if m else html
+    for marker in ("sharedaddy", "jp-relatedposts", "comments"):
+        i = body.find(marker)
+        if i > 400:
+            body = body[:i]
+            break
+    return strip_tags(body)[:8000]
+
+
+def main() -> None:
+    store = {
+        "meta": {
+            "system": "ogham",
+            "note": "Пары огама в источниках не таблицируются; сочетания — соседство fews в раскладе, сводятся из значений.",
+        },
+        "cards": {},
+        "errors": [],
+    }
+    for cid, en, tree, _p in FEDA:
+        store["cards"][cid] = {
+            "id": cid,
+            "name_en": en,
+            "tree_ru": tree,
+            "meanings": [],
+        }
+
+    print("== oghamlore ==")
+    st, html = fetch("https://oghamlore.com/learn/ogham-divination/")
+    lore = parse_oghamlore(html) if st == 200 else {}
+    print("  parsed", list(lore.keys()))
+    for cid, en, tree, _p in FEDA:
+        if en in lore:
+            store["cards"][cid]["meanings"].append(
+                {"source": "oghamlore", "url": "https://oghamlore.com/learn/ogham-divination/", "text": lore[en]}
+            )
+
+    print("== druidogham ==")
+    for cid, en, tree, path in FEDA:
+        url = f"https://druidogham.wordpress.com/{path}/"
+        st, html = fetch(url)
+        if st != 200:
+            store["errors"].append({"url": url, "status": st})
+            print("  FAIL", cid, st)
+            continue
+        text = wp_content(html)
+        store["cards"][cid]["meanings"].append({"source": "druidogham", "url": url, "text": text})
+        print(f"  {cid:8} chars={len(text)}")
+
+    print("== globalspiritualstudies chart ==")
+    st, html = fetch("https://globalspiritualstudies.com/free-ogham-alphabet-chart/")
+    if st == 200:
+        text = strip_tags(html)
+        store["meta"]["chart"] = text[:12000]
+        print("  chart chars", len(text))
+
+    nmean = sum(1 for c in store["cards"].values() if c["meanings"])
+    CORPUS.parent.mkdir(parents=True, exist_ok=True)
+    CORPUS.write_text(json.dumps(store, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print("wrote", CORPUS, "fews_with_meanings", nmean, "errors", len(store["errors"]))
+
+
+if __name__ == "__main__":
+    main()