feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets

2026-10-09 15:26:58 +05:00
parent 7f91366201
commit 0b12e4583b
244 changed files with 611461 additions and 1617 deletions
@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Скрипт инициализации и миграции базы данных 1000 Паспортов Знаков «Оракула 999».
+Создает assets/data/oracle_passports.json и связывает все 78 полей
+с существующими данными (words_data.dart, oracle_kb.json, traditions_canon.json).
+"""
+
+import os
+import sys
+import json
+import re
+from pathlib import Path
+from datetime import datetime
+
+sys.path.insert(0, os.path.abspath("oracle_enricher"))
+from enricher.passport_schema import (
+    PROMPTS_78,
+    empty_passport_record,
+    render_passport_markdown,
+    count_filled_fields,
+)
+
+DART_WORDS_PATH = "lib/data/words_data.dart"
+KB_PATH = "assets/data/oracle_kb.json"
+TRADITIONS_PATH = "assets/data/traditions_canon.json"
+PASSPORTS_OUT_PATH = "assets/data/oracle_passports.json"
+PASSPORTS_ENRICHER_PATH = "oracle_enricher/data/oracle_passports.json"
+
+
+def load_words_from_dart():
+    with open(DART_WORDS_PATH, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    pattern = re.compile(
+        r'WordItem\s*\(\s*id:\s*(\d+),\s*word:\s*"([^"]+)",\s*partOfSpeech:\s*"([^"]+)",\s*frequency:\s*([\d\.]+),\s*category:\s*"([^"]+)",\s*description:\s*"(.*?)",\s*\)',
+        re.DOTALL
+    )
+    words = []
+    for m in pattern.findall(content):
+        wid = int(m[0])
+        words.append({
+            "id": wid,
+            "word": m[1].strip(),
+            "pos": m[2].strip(),
+            "frequency": float(m[3]),
+            "category": m[4].strip(),
+            "description": m[5].strip(),
+        })
+    return words
+
+
+def load_kb():
+    if not os.path.exists(KB_PATH):
+        return {}
+    with open(KB_PATH, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    # Нормализуем ключи: по слову и по id
+    kb_map = {}
+    for k, v in data.items():
+        if isinstance(v, dict):
+            kb_map[k.lower().strip()] = v
+            if "id" in v:
+                kb_map[str(v["id"])] = v
+    return kb_map
+
+
+def load_traditions():
+    if not os.path.exists(TRADITIONS_PATH):
+        return {}
+    with open(TRADITIONS_PATH, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def build_tradition_lookup(traditions_data):
+    """Индексирует станции традиций по словам."""
+    lookup = {}
+    for canon_id, canon in traditions_data.items():
+        if not isinstance(canon, dict):
+            continue
+        canon_name = canon.get("name", canon_id)
+        stations = canon.get("stations", [])
+        for st in stations:
+            st_name = st.get("name", "")
+            st_symbol = st.get("symbol", "")
+            st_lore = st.get("lore", "")
+            
+            # 1. Primary
+            prim = st.get("primary")
+            if isinstance(prim, dict) and prim.get("word"):
+                pw = str(prim["word"]).lower().strip()
+                lookup.setdefault(pw, []).append({
+                    "canon_id": canon_id,
+                    "canon_name": canon_name,
+                    "station_name": st_name,
+                    "symbol": st_symbol,
+                    "lore": st_lore,
+                    "is_primary": True,
+                })
+            elif isinstance(prim, str) and prim.strip():
+                pw = prim.lower().strip()
+                lookup.setdefault(pw, []).append({
+                    "canon_id": canon_id,
+                    "canon_name": canon_name,
+                    "station_name": st_name,
+                    "symbol": st_symbol,
+                    "lore": st_lore,
+                    "is_primary": True,
+                })
+                
+            # 2. Cluster / Intersections
+            cluster = st.get("cluster", [])
+            if isinstance(cluster, list):
+                for cl in cluster:
+                    if isinstance(cl, dict) and cl.get("word"):
+                        cw = str(cl["word"]).lower().strip()
+                        lookup.setdefault(cw, []).append({
+                            "canon_id": canon_id,
+                            "canon_name": canon_name,
+                            "station_name": st_name,
+                            "symbol": st_symbol,
+                            "lore": st_lore,
+                            "is_primary": False,
+                        })
+                    elif isinstance(cl, str) and cl.strip():
+                        cw = cl.lower().strip()
+                        lookup.setdefault(cw, []).append({
+                            "canon_id": canon_id,
+                            "canon_name": canon_name,
+                            "station_name": st_name,
+                            "symbol": st_symbol,
+                            "lore": st_lore,
+                            "is_primary": False,
+                        })
+    return lookup
+
+
+def main():
+    print("=" * 70)
+    print("ИНИЦИАЛИЗАЦИЯ И МИГРАЦИЯ БАЗЫ ДАННЫХ 1000 ПАСПОРТОВ ОРАКУЛА 999")
+    print("=" * 70)
+
+    words = load_words_from_dart()
+    print(f"Загружено слов из Dart: {len(words)}")
+
+    kb = load_kb()
+    print(f"Загружено записей базы знаний oracle_kb: {len(kb)}")
+
+    traditions = load_traditions()
+    trad_lookup = build_tradition_lookup(traditions)
+    print(f"Проиндексировано слов с мантическими станциями: {len(trad_lookup)}")
+
+    # Если уже есть существующая база паспортов — загружаем для объединения
+    existing_passports = {}
+    if os.path.exists(PASSPORTS_OUT_PATH):
+        try:
+            with open(PASSPORTS_OUT_PATH, "r", encoding="utf-8") as f:
+                raw_p = json.load(f)
+                if isinstance(raw_p, dict):
+                    existing_passports = {str(v.get("id", k)): v for k, v in raw_p.items()}
+                elif isinstance(raw_p, list):
+                    existing_passports = {str(item["id"]): item for item in raw_p}
+            print(f"Найдено существующих паспортов в базе: {len(existing_passports)}")
+        except Exception as e:
+            print(f"Ошибка чтения старых паспортов: {e}")
+
+    passports_db = {}
+    total_filled_across_db = 0
+
+    for w in words:
+        wid = w["id"]
+        wid_str = str(wid)
+        id_pad = str(wid).zfill(3)
+        word_clean = w["word"].lower().strip()
+
+        kb_entry = kb.get(word_clean) or kb.get(wid_str) or {}
+        existing_p = existing_passports.get(wid_str) or {}
+
+        # Базовая структура
+        record = empty_passport_record({
+            "id": wid,
+            "word": w["word"],
+            "pos": w["pos"],
+            "category": w["category"],
+            "emoji": kb_entry.get("emoji", existing_p.get("emoji", "")),
+        })
+
+        # Если уже были сохраненные поля из предыдущих запусков
+        if "fields" in existing_p:
+            for k, v in existing_p["fields"].items():
+                if v and v.strip():
+                    record["fields"][k] = v.strip()
+
+        f = record["fields"]
+
+        # ── РАЗДЕЛ I ──
+        if not f.get("index_and_name"):
+            emojis = record["emoji"] or "✦"
+            f["index_and_name"] = f"{id_pad} | {w['word'].upper()} | {emojis}"
+        if not f.get("grammatical_class"):
+            f["grammatical_class"] = w["pos"]
+        if not f.get("functional_category"):
+            f["functional_category"] = w["category"]
+
+        # Сетевой статус
+        stations = trad_lookup.get(word_clean, [])
+        if not f.get("network_status"):
+            if stations:
+                primary_names = [f"{s['canon_name']}: {s['station_name']}" for s in stations if s["is_primary"]]
+                cross_names = [f"{s['canon_name']}: {s['station_name']}" for s in stations if not s["is_primary"]]
+                if primary_names:
+                    f["network_status"] = f"Узловая станция (Первичное слово: {', '.join(primary_names[:2])})"
+                else:
+                    f["network_status"] = f"Узловая станция (Пересечение: {', '.join(cross_names[:2])})"
+            else:
+                f["network_status"] = "Свободный знак поля"
+
+        # ── РАЗДЕЛ II: Кросс-мантическая сеть ──
+        trad_mapping = {
+            "runes": "mantic_runes",
+            "ogham": "mantic_ogham",
+            "tarot": "mantic_tarot",
+            "lenormand": "mantic_lenormand",
+            "kabbalah": "mantic_kabbalah",
+            "geomancy": "mantic_geomancy",
+            "iching": "mantic_iching",
+            "alchemy": "mantic_alchemy",
+            "western_zodiac": "mantic_zodiac_west",
+            "zodiac": "mantic_zodiac_chinese",
+            "days_of_week": "mantic_septener",
+            "months_of_year": "mantic_dodecade",
+            "numerology": "mantic_numerology",
+            "cosmology": "mantic_cosmology",
+        }
+
+        station_laws_acc = []
+        for s in stations:
+            c_id = s["canon_id"]
+            field_key = trad_mapping.get(c_id)
+            if field_key and not f.get(field_key):
+                role_str = "Первичное слово" if s["is_primary"] else "Пересечение"
+                f[field_key] = f"[{s['station_name']} {s['symbol']} / Роль: {role_str} / Сакральный узел]"
+            if s.get("lore"):
+                station_laws_acc.append(f"{s['station_name']}: {s['lore']}")
+
+        # Слои (layers) из базы знаний
+        for l in kb_entry.get("layers", []):
+            sys_name = l.get("system", "").lower()
+            title = l.get("title", "")
+            law = l.get("law", "")
+            if "рун" in sys_name and not f.get("mantic_runes"):
+                f["mantic_runes"] = f"[{title} / Роль: Слой / {law[:60]}]"
+            elif "таро" in sys_name and not f.get("mantic_tarot"):
+                f["mantic_tarot"] = f"[{title} / Роль: Слой / {law[:60]}]"
+            elif "огам" in sys_name and not f.get("mantic_ogham"):
+                f["mantic_ogham"] = f"[{title} / Роль: Слой / {law[:60]}]"
+            elif "ленорман" in sys_name and not f.get("mantic_lenormand"):
+                f["mantic_lenormand"] = f"[{title} / Роль: Слой / {law[:60]}]"
+            elif "каббал" in sys_name and not f.get("mantic_kabbalah"):
+                f["mantic_kabbalah"] = f"[{title} / Роль: Слой / {law[:60]}]"
+            elif "геомант" in sys_name and not f.get("mantic_geomancy"):
+                f["mantic_geomancy"] = f"[{title} / Роль: Слой / {law[:60]}]"
+            elif "и цзин" in sys_name and not f.get("mantic_iching"):
+                f["mantic_iching"] = f"[{title} / Роль: Слой / {law[:60]}]"
+            elif "алхим" in sys_name and not f.get("mantic_alchemy"):
+                f["mantic_alchemy"] = f"[{title} / Роль: Слой / {law[:60]}]"
+
+        if not f.get("mantic_station_laws") and station_laws_acc:
+            f["mantic_station_laws"] = " • ".join(station_laws_acc[:3])
+
+        # ── РАЗДЕЛ III: Языки и графика ──
+        kb_langs = kb_entry.get("languages", {})
+        lang_mapping = {
+            "шумерский": "script_sumer_akkad",
+            "древнеегипетский": "script_egypt",
+            "иврит": "script_hebrew",
+            "древнегреческий": "script_antiquity",
+            "латынь": "script_antiquity",
+            "древнеирландский": "script_celtic",
+            "древнескандинавский": "script_runic_germanic",
+            "старославянский": "script_slavic",
+            "санскрит": "script_india_sanskrit",
+            "авестийский": "script_persia",
+            "древнекитайский": "script_china",
+            "японский": "script_japan",
+            "арабский": "script_islamic",
+        }
+        for lang_name, lang_val in kb_langs.items():
+            field_k = lang_mapping.get(lang_name.lower())
+            if field_k and not f.get(field_k) and lang_val:
+                f[field_k] = str(lang_val)
+
+        # ── РАЗДЕЛ IV: Трехполярная матрица ──
+        if not f.get("state_neutral"):
+            clean_desc = re.sub(r'\[.*?\]\s*', '', w['description']).strip()
+            if clean_desc:
+                f["state_neutral"] = clean_desc
+        if not f.get("state_creative") and kb_entry.get("core"):
+            f["state_creative"] = kb_entry["core"]
+        if not f.get("state_destructive") and kb_entry.get("shadow"):
+            f["state_destructive"] = kb_entry["shadow"]
+
+        # ── РАЗДЕЛ V: Культуры ──
+        kb_cults = kb_entry.get("cultures", {})
+        cult_mapping = {
+            "Шумер и Месопотамия": ("culture_sumer_light", "culture_sumer_shadow"),
+            "Древний Египет": ("culture_egypt_light", "culture_egypt_shadow"),
+            "Иудейская традиция и Каббала": ("culture_kabbalah_light", "culture_kabbalah_shadow"),
+            "Античность (Греция и Рим)": ("culture_antiquity_light", "culture_antiquity_shadow"),
+            "Кельты и друиды": ("culture_celtic_light", "culture_celtic_shadow"),
+            "Скандинавия и германцы": ("culture_nordic_light", "culture_nordic_shadow"),
+            "Славянская традиция": ("culture_slavic_light", "culture_slavic_shadow"),
+            "Ведическая Индия": ("culture_india_light", "culture_india_shadow"),
+            "Персия и зороастризм": ("culture_persia_light", "culture_persia_shadow"),
+            "Древний Китай": ("culture_china_light", "culture_china_shadow"),
+            "Япония и синто": ("culture_japan_light", "culture_japan_shadow"),
+            "Ислам и суфизм": ("culture_islam_light", "culture_islam_shadow"),
+            "Алхимия и герметизм": ("culture_hermetic_light", "culture_hermetic_shadow"),
+            "Мезоамерика и Анды": ("culture_mesoamerica_light", "culture_mesoamerica_shadow"),
+            "Полинезия и Океания": ("culture_polynesia_light", "culture_polynesia_shadow"),
+        }
+        for cult_name, val in kb_cults.items():
+            pair = cult_mapping.get(cult_name)
+            if pair:
+                light_k, shadow_k = pair
+                if not f.get(light_k) and val:
+                    f[light_k] = str(val)[:250]
+
+        # ── РАЗДЕЛ VI: Синтез ──
+        if not f.get("semantic_core") and kb_entry.get("core"):
+            f["semantic_core"] = kb_entry["core"]
+        if not f.get("will_vector") and kb_entry.get("asks"):
+            f["will_vector"] = kb_entry["asks"]
+        if not f.get("shadow_warning") and kb_entry.get("shadow"):
+            f["shadow_warning"] = kb_entry["shadow"]
+
+        # Подсчет заполненных
+        filled, total_78 = count_filled_fields(record)
+        record["filled_count"] = filled
+        record["is_complete"] = (filled == total_78)
+        record["updated_at"] = datetime.now().isoformat()
+        total_filled_across_db += filled
+
+        passports_db[str(wid)] = record
+
+    # Сохраняем в файл
+    os.makedirs(os.path.dirname(PASSPORTS_OUT_PATH), exist_ok=True)
+    with open(PASSPORTS_OUT_PATH, "w", encoding="utf-8") as f_out:
+        json.dump(passports_db, f_out, ensure_ascii=False, indent=2)
+
+    # Дублируем в oracle_enricher/data/
+    os.makedirs(os.path.dirname(PASSPORTS_ENRICHER_PATH), exist_ok=True)
+    with open(PASSPORTS_ENRICHER_PATH, "w", encoding="utf-8") as f_out:
+        json.dump(passports_db, f_out, ensure_ascii=False, indent=2)
+
+    avg_filled = total_filled_across_db / len(words)
+    print("\n" + "=" * 70)
+    print("ИТОГИ ИНИЦИАЛИЗАЦИИ БАЗЫ ПАСПОРТОВ 000–999:")
+    print(f"Всего паспортов в базе: {len(passports_db)}")
+    print(f"Всего заполненных параметров по всей базе: {total_filled_across_db} из {len(words)*78}")
+    print(f"Среднее заполнение на одну карточку: {avg_filled:.1f} из 78 полей")
+    print(f"Файл базы сохранен в: {PASSPORTS_OUT_PATH} и {PASSPORTS_ENRICHER_PATH}")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    main()