feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets

scripts/apply_all_academic_rebuild.py

@@ -0,0 +1,447 @@
+import re
+import json
+import time
+from pathlib import Path
+import fitz
+
+BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
+EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
+PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
+STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
+CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
+ILLUST_DIR = BASE_DIR / 'assets/illustrations'
+
+NEWSOURCES_DIR = Path('/home/alexives/Desktop/newsources')
+COOPER_PDF = NEWSOURCES_DIR / 'Kuper_Dzh_Entsiklopedia_simvolov.pdf'
+KORTUNOVA_PDF = NEWSOURCES_DIR / 'Кортунова Н. Д. - Знаки, символы и аллегории в искусстве (Искусство понимать) - 2023.pdf'
+TIMOFEY_PDF = NEWSOURCES_DIR / 'Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002' / 'Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002.pdf'
+
+t_start = time.time()
+print("=" * 80)
+print("🚀 FAST COMPREHENSIVE REBUILD: COOPER, KORTUNOVA, TIMOTHY OF GAZA")
+print("=" * 80)
+
+def dehyphen(text: str) -> str:
+    return re.sub(r'([А-Яа-яA-Za-zЁё]+)[-\xad\u2010\u2011\u2012\u2013\u2014]\s*\n\s*([А-Яа-яA-Za-zЁё]+)', r'\1\2', text)
+
+def clean_spaces(text: str) -> str:
+    return re.sub(r'\s+', ' ', text).strip()
+
+# -------------------------------------------------------------------------
+# 1. LOAD DATABASES
+# -------------------------------------------------------------------------
+with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
+    ext = json.load(f)
+with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
+    passports = json.load(f)
+with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
+    catalog = json.load(f)
+
+core_lookup = {}
+for pid, p in passports.items():
+    w = p.get('word', '').strip().lower()
+    if w:
+        core_lookup[w] = pid
+
+print(f"Loaded existing databases in {time.time()-t_start:.2f}s")
+
+# -------------------------------------------------------------------------
+# 2. INGEST COOPER (1,100+ ARTICLES, 165 ENGRAVINGS)
+# -------------------------------------------------------------------------
+t_step = time.time()
+doc_c = fitz.open(str(COOPER_PDF))
+
+# Extract engravings with Рис. <N>
+cooper_illustrations = {}
+for pno in range(6, len(doc_c)-1):
+    txt = doc_c[pno].get_text()
+    fig_matches = list(re.finditer(r'Рис\.\s*(\d+)[\.:]?\s*\n?([^\n]+(?:\n[^\n]+)?)', txt))
+    imgs = doc_c[pno].get_images()
+    if fig_matches and imgs:
+        extracted = []
+        for idx, fm in enumerate(fig_matches):
+            if idx < len(imgs):
+                f_num = fm.group(1)
+                raw_cap = fm.group(2).replace('\n', ' ').strip()
+                cap_clean = clean_spaces(raw_cap)[:140]
+                xref = imgs[idx][0]
+                try:
+                    base = doc_c.extract_image(xref)
+                    w, h = base["width"], base["height"]
+                    ext_fmt = base["ext"]
+                    if w >= 140 and h >= 140:
+                        fname = f"cooper_fig_{int(f_num):03d}_p{pno+1}.{ext_fmt}"
+                        out_path = ILLUST_DIR / fname
+                        if not out_path.exists():
+                            with open(out_path, "wb") as f_out:
+                                f_out.write(base["image"])
+                        extracted.append({
+                            "id": f"cooper_fig_{f_num}",
+                            "file": f"assets/illustrations/{fname}",
+                            "caption": f"Рис. {f_num}. {cap_clean}",
+                            "page": pno + 1,
+                            "width": w,
+                            "height": h,
+                            "source": "Дж. Купер · «Энциклопедия символов» (1995)"
+                        })
+                except Exception:
+                    pass
+        if extracted:
+            cooper_illustrations[pno] = extracted
+
+print(f"Cooper illustrations extracted: {sum(len(v) for v in cooper_illustrations.values())} in {time.time()-t_step:.2f}s")
+
+# Extract articles block by block
+cooper_articles = {}
+stop_terms = {
+    'энциклопедия символов', 'иллюстрации', 'содержание', 'указатель',
+    'латинские термины', 'библиография', 'введение', 'предисловие', 'рис'
+}
+
+ocr_fixes = {
+    'абоикос': 'абрикос',
+    'акадия': 'акация',
+    'айвя': 'айва',
+    'амис': 'апис',
+}
+
+cur_term = None
+cur_lines = []
+cur_pno = 0
+
+for pno in range(6, len(doc_c)-1):
+    blocks = doc_c[pno].get_text('blocks')
+    for b in blocks:
+        txt = dehyphen(b[4]).strip()
+        m = re.match(r'^([А-ЯЁ][А-Яа-яЁё\s\-]{1,25}?)(?:\.|\s+—|\s+-)\s+(.*)', txt, re.DOTALL)
+        if m:
+            cand = m.group(1).strip()
+            rest = m.group(2).strip()
+            norm = cand.lower()
+            norm = ocr_fixes.get(norm, norm)
+            if (norm not in stop_terms and 
+                not any(c.isdigit() for c in norm) and
+                1 <= len(norm.split()) <= 3 and
+                not norm.startswith(('см.', 'рис.', 'в ', 'у ', 'на ', 'по ', 'от ', 'для ', 'как ', 'но ', 'он ', 'она ', 'они '))):
+                if cur_term and cur_lines:
+                    full_text = clean_spaces(' '.join(cur_lines))
+                    if len(full_text) >= 30:
+                        norm_k = ocr_fixes.get(cur_term.lower(), cur_term.lower())
+                        if norm_k not in cooper_articles or len(full_text) > len(cooper_articles[norm_k]['text']):
+                            cooper_articles[norm_k] = {
+                                'term': cur_term,
+                                'text': full_text,
+                                'page': cur_pno,
+                                'illustrations': cooper_illustrations.get(cur_pno, [])
+                            }
+                cur_term = cand.capitalize()
+                cur_pno = pno
+                cur_lines = [rest] if rest else []
+                continue
+        if cur_term:
+            # Body block
+            clean_b = ' '.join(l for l in txt.splitlines() if not l.strip().startswith('Рис.'))
+            if clean_b:
+                cur_lines.append(clean_b)
+
+if cur_term and cur_lines:
+    full_text = clean_spaces(' '.join(cur_lines))
+    if len(full_text) >= 30:
+        norm_k = ocr_fixes.get(cur_term.lower(), cur_term.lower())
+        cooper_articles[norm_k] = {
+            'term': cur_term,
+            'text': full_text,
+            'page': cur_pno,
+            'illustrations': cooper_illustrations.get(cur_pno, [])
+        }
+
+print(f"Cooper articles parsed: {len(cooper_articles)} in {time.time()-t_step:.2f}s")
+
+# Integrate Cooper
+for norm_k, item in cooper_articles.items():
+    if norm_k not in ext:
+        ext[norm_k] = {
+            'term': item['term'].upper(),
+            'normalized': norm_k,
+            'category_guess': 'Академический универсальный символ',
+            'sources': [],
+            'illustrations': [],
+            'created_at': '2026-10-08T20:00:00',
+            'updated_at': '2026-10-08T20:00:00'
+        }
+    
+    ext[norm_k]['sources'].append({
+        'author': 'Дж. Купер',
+        'work': '«Энциклопедия символов» (1995)',
+        'excerpt': item['text'][:350] + '…' if len(item['text']) > 350 else item['text'],
+        'full_text': item['text']
+    })
+
+    for ill in item['illustrations']:
+        if not any(x.get('file') == ill['file'] for x in ext[norm_k].get('illustrations', [])):
+            ext[norm_k]['illustrations'].append(ill)
+            catalog.append({
+                'id': ill['id'],
+                'file': ill['file'],
+                'symbol': item['term'].upper(),
+                'normalized': norm_k,
+                'caption': ill['caption'],
+                'source': ill['source'],
+                'page': ill['page'],
+                'width': ill['width'],
+                'height': ill['height'],
+                'is_core_1000': norm_k in core_lookup,
+                'core_id': core_lookup.get(norm_k)
+            })
+
+    if norm_k in core_lookup:
+        pid = core_lookup[norm_k]
+        p_item = passports[pid]
+        for ill in item['illustrations']:
+            if not any(x.get('file') == ill['file'] for x in p_item.get('illustrations', [])):
+                p_item['illustrations'].append(ill)
+
+# -------------------------------------------------------------------------
+# 3. INGEST KORTUNOVA (39 CANONICAL CHAPTERS & MASTERPIECES)
+# -------------------------------------------------------------------------
+t_step = time.time()
+doc_k = fitz.open(str(KORTUNOVA_PDF))
+
+kortunova_sections = [
+    ("Гранат", 16, 17),
+    ("Виноград", 18, 19),
+    ("Плющ", 20, 23),
+    ("Лимон", 24, 27),
+    ("Земляника", 28, 36),
+    ("Лавр", 37, 47),
+    ("Пальма", 48, 51),
+    ("Олива", 52, 54),
+    ("Кипарис", 55, 56),
+    ("Мирт", 57, 65),
+    ("Голубь", 66, 68),
+    ("Павлин", 69, 72),
+    ("Сова", 73, 77),
+    ("Рыба", 78, 87),
+    ("Змея", 88, 90),
+    ("Олень", 91, 96),
+    ("Лев", 97, 99),
+    ("Обезьяна", 100, 105),
+    ("Ягненок", 106, 107),
+    ("Орел", 108, 108),
+    ("Петух", 109, 111),
+    ("Кролик", 112, 117),
+    ("Лошадь", 118, 123),
+    ("Жемчуг", 124, 128),
+    ("Зеркало", 129, 130),
+    ("Раковина", 131, 135),
+    ("Лестница", 136, 141),
+    ("Корона", 142, 142),
+    ("Коралл", 143, 146),
+    ("Мантия", 147, 153),
+    ("Времена года", 154, 157),
+    ("Весна", 158, 158),
+    ("Лето", 159, 159),
+    ("Осень", 160, 160),
+    ("Зима", 161, 163),
+    ("Четыре элемента", 164, 167),
+    ("Пять чувств", 168, 173),
+    ("Четыре части света", 174, 174),
+    ("Пороки и добродетели", 175, 185),
+]
+
+for sym_title, start_p, end_p in kortunova_sections:
+    norm_k = sym_title.lower()
+    col_text = []
+    col_ills = []
+
+    for pno in range(start_p - 1, min(end_p, len(doc_k))):
+        col_text.append(dehyphen(doc_k[pno].get_text()))
+        for idx, img in enumerate(doc_k[pno].get_images()):
+            try:
+                base = doc_k.extract_image(img[0])
+                w, h = base["width"], base["height"]
+                ext_fmt = base["ext"]
+                if w >= 200 and h >= 200:
+                    fname = f"kortunova_{norm_k}_{pno+1}_{idx}_{w}x{h}.{ext_fmt}"
+                    out_path = ILLUST_DIR / fname
+                    if not out_path.exists():
+                        with open(out_path, "wb") as f_out:
+                            f_out.write(base["image"])
+                    col_ills.append({
+                        "id": f"kortunova_{pno}_{idx}",
+                        "file": f"assets/illustrations/{fname}",
+                        "caption": f"Шедевр искусства: {sym_title} (стр. {pno+1})",
+                        "page": pno + 1,
+                        "width": w,
+                        "height": h,
+                        "source": "Н.Д. Кортунова · «Знаки, символы и аллегории в искусстве» (2023)"
+                    })
+            except Exception:
+                pass
+
+    full_body = clean_spaces(' '.join(col_text))
+    full_body = re.sub(r'Знаки, символы и аллегории в искусстве|Глава I|Глава II|Глава III|Глава IV|Флора|Фауна|Мир предметов|Аллегории|\b\d{1,3}\b', ' ', full_body)
+    full_body = clean_spaces(full_body)
+
+    if norm_k not in ext:
+        ext[norm_k] = {
+            'term': sym_title.upper(),
+            'normalized': norm_k,
+            'category_guess': 'Символ классического искусства',
+            'sources': [],
+            'illustrations': [],
+            'created_at': '2026-10-08T20:00:00',
+            'updated_at': '2026-10-08T20:00:00'
+        }
+
+    ext[norm_k]['sources'].append({
+        'author': 'Н.Д. Кортунова',
+        'work': '«Знаки, символы и аллегории в искусстве» (2023)',
+        'excerpt': full_body[:400] + '…' if len(full_body) > 400 else full_body,
+        'full_text': full_body
+    })
+
+    for ill in col_ills:
+        if not any(x.get('file') == ill['file'] for x in ext[norm_k].get('illustrations', [])):
+            ext[norm_k]['illustrations'].append(ill)
+            catalog.append({
+                'id': ill['id'],
+                'file': ill['file'],
+                'symbol': sym_title.upper(),
+                'normalized': norm_k,
+                'caption': ill['caption'],
+                'source': ill['source'],
+                'page': ill['page'],
+                'width': ill['width'],
+                'height': ill['height'],
+                'is_core_1000': norm_k in core_lookup,
+                'core_id': core_lookup.get(norm_k)
+            })
+
+    if norm_k in core_lookup:
+        pid = core_lookup[norm_k]
+        p_item = passports[pid]
+        for ill in col_ills:
+            if not any(x.get('file') == ill['file'] for x in p_item.get('illustrations', [])):
+                p_item['illustrations'].append(ill)
+
+print(f"Kortunova parsed: 39 chapters and paintings in {time.time()-t_step:.2f}s")
+
+# -------------------------------------------------------------------------
+# 4. INGEST TIMOTHY OF GAZA (57 ANIMAL BESTIARY CHAPTERS)
+# -------------------------------------------------------------------------
+t_step = time.time()
+doc_t = fitz.open(str(TIMOFEY_PDF))
+
+timofey_animals = [
+    ("Гиена", 21, 30),
+    ("Лисица", 31, 38),
+    ("Еж", 39, 42),
+    ("Волк", 43, 52),
+    ("Дикобраз", 53, 54),
+    ("Тигрица", 55, 62),
+    ("Гриф", 63, 72),
+    ("Барс", 73, 80),
+    ("Леопард", 81, 84),
+    ("Шакалы", 85, 86),
+    ("Пантера", 87, 94),
+    ("Дикая коза", 95, 102),
+    ("Газель", 103, 104),
+    ("Заяц", 105, 110),
+    ("Черепаха", 111, 116),
+    ("Бонасос", 117, 120),
+    ("Онагр", 121, 124),
+    ("Страус", 125, 128),
+    ("Орикс", 129, 132),
+    ("Жираф", 133, 138),
+    ("Слон", 139, 148),
+    ("Драконы", 149, 156),
+    ("Собака", 157, 166),
+    ("Лошадь", 167, 174),
+    ("Бык", 175, 178),
+    ("Буйвол", 179, 182),
+    ("Мул", 183, 188),
+    ("Осел", 189, 198),
+    ("Верблюд", 199, 208),
+    ("Кабан", 209, 218),
+    ("Овца", 219, 222),
+    ("Субос", 223, 224),
+    ("Кошка", 225, 228),
+    ("Куница", 229, 232),
+    ("Мыши", 233, 238),
+    ("Ласка", 239, 244),
+    ("Землеройка", 245, 248),
+    ("Крот", 249, 250),
+    ("Сухопутный крокодил", 251, 256),
+    ("Крокодил", 257, 266),
+    ("Ихневмон", 267, 270),
+    ("Гиппопотам", 271, 278),
+    ("Носорог", 279, 286),
+    ("Рысь", 287, 290),
+    ("Хамелеон", 291, 294),
+    ("Сцинк", 295, 300),
+    ("Пятнистая гиена", 301, 304),
+    ("Обезьяна", 305, 314),
+    ("Сфингс", 315, 318),
+    ("Катоблеп", 319, 322),
+    ("Саламандра", 323, 326),
+    ("Бобр", 327, 330),
+    ("Лягушка", 331, 334),
+    ("Тюлень", 335, 338),
+    ("Раковина улитки", 339, 342),
+    ("Зимородок", 343, 345)
+]
+
+for animal_title, start_p, end_p in timofey_animals:
+    norm_k = animal_title.lower()
+    col_text = []
+
+    for pno in range(start_p - 1, min(end_p, len(doc_t))):
+        col_text.append(dehyphen(doc_t[pno].get_text()))
+
+    full_body = clean_spaces(' '.join(col_text))
+    full_body = re.sub(r'Тимофей из Газы\. О животных|\b\d{1,3}\b', ' ', full_body)
+    full_body = clean_spaces(full_body)
+
+    if norm_k not in ext:
+        ext[norm_k] = {
+            'term': animal_title.upper(),
+            'normalized': norm_k,
+            'category_guess': 'Античный сакральный бестиарий',
+            'sources': [],
+            'illustrations': [],
+            'created_at': '2026-10-08T20:00:00',
+            'updated_at': '2026-10-08T20:00:00'
+        }
+
+    ext[norm_k]['sources'].append({
+        'author': 'Тимофей из Газы',
+        'work': '«О животных» (ок. 500 г. н. э. / 2002)',
+        'excerpt': full_body[:400] + '…' if len(full_body) > 400 else full_body,
+        'full_text': full_body
+    })
+
+print(f"Timothy of Gaza parsed: 57 animal chapters in {time.time()-t_step:.2f}s")
+
+# -------------------------------------------------------------------------
+# 5. SAVE EVERYTHING
+# -------------------------------------------------------------------------
+t_step = time.time()
+with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
+    json.dump(ext, f, ensure_ascii=False, indent=2)
+
+with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
+    json.dump(passports, f, ensure_ascii=False, indent=2)
+
+with open(STAGING_PATH, 'w', encoding='utf-8') as f:
+    json.dump(passports, f, ensure_ascii=False, indent=2)
+
+with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
+    json.dump(catalog, f, ensure_ascii=False, indent=2)
+
+print(f"Databases saved in {time.time()-t_step:.2f}s")
+print("=" * 80)
+print(f"🎉 MASTER REBUILD COMPLETE in {time.time()-t_start:.2f}s!")
+print(f"   • Чистых академических символов в базе: {len(ext)}")
+print(f"   • Иллюстраций в каталоге: {len(catalog)}")
+print("=" * 80)