728 lines
40 KiB
Python
Executable File
728 lines
40 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
ingest_secondary_deep_corpus.py
|
||
===============================
|
||
Глубокий парсер академических источников из /home/alexives/Desktop/newsources:
|
||
1. Дж. Купер — «Энциклопедия символов» (1995, 397 стр.) + 250+ гравюр
|
||
2. В.В. Карелин — «Лабиринт. Дракон. Единорог… О чем говорят символы» (2015, 163 стр.) + иллюстрации
|
||
3. Н.Д. Кортунова — «Знаки, символы и аллегории в искусстве» (2023, 193 стр.) + шедевры живописи
|
||
4. К.Г. Юнг — 3 тома: «Символы трансформации», «Символы сновидений», «Человек и его символы»
|
||
5. В.В. Байдин — «Архетипы и символы русской культуры» (2021, FB2)
|
||
6. Б. Гофф — «Symbols of Prehistoric Mesopotamia» (1963, 498 стр.)
|
||
7. Тимофей из Газы — «Тигрица и грифон. Сакральные символы животного мира» (2002)
|
||
8. Р. Штайнер — «Оккультные знаки и символы» (FB2)
|
||
|
||
ПРАВИЛА:
|
||
- Приоритет: 1000 знаков Оракула (000–999). Каждая культура ложится в соответствующую ячейку Раздела V.
|
||
- Все неучтенное: попадает в oracle_external_symbols.json.
|
||
- Картинки: обязательны, сохраняются в assets/illustrations/ и привязываются к объектам.
|
||
- Цитаты: строго аутентичные, в формате [Автор · «Книга»]: «...».
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import json
|
||
import re
|
||
import io
|
||
from pathlib import Path
|
||
from typing import Dict, Any, List, Optional, Tuple, Set
|
||
from datetime import datetime
|
||
from PIL import Image
|
||
|
||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(PROJECT_ROOT))
|
||
|
||
import fitz
|
||
|
||
from oracle_enricher.enricher.passport_schema import count_filled_fields
|
||
|
||
NEWSOURCES_DIR = Path("/home/alexives/Desktop/newsources")
|
||
ILLUSTRATIONS_DIR = PROJECT_ROOT / "assets" / "illustrations"
|
||
CATALOG_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_illustrations_catalog.json"
|
||
STAGING_DB_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_passports_staging.json"
|
||
EXTERNAL_DB_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_external_symbols.json"
|
||
ENRICHER_EXTERNAL_PATH = PROJECT_ROOT / "oracle_enricher" / "data" / "oracle_external_symbols.json"
|
||
|
||
|
||
def clean_spaces(text: str) -> str:
|
||
text = re.sub(r'(\w+)-\s*\n\s*(\w+)', r'\1\2', text)
|
||
return re.sub(r'\s+', ' ', text).strip()
|
||
|
||
|
||
def transliterate_to_slug(text: str) -> str:
|
||
ru_to_en = {
|
||
'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo',
|
||
'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm',
|
||
'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u',
|
||
'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'shch',
|
||
'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya'
|
||
}
|
||
slug = []
|
||
for c in text.lower():
|
||
if c in ru_to_en:
|
||
slug.append(ru_to_en[c])
|
||
elif c.isalnum():
|
||
slug.append(c)
|
||
elif c in [' ', '-', '_']:
|
||
slug.append('_')
|
||
res = re.sub(r'_+', '_', ''.join(slug)).strip('_')
|
||
return res[:30] if res else 'symbol'
|
||
|
||
|
||
def extract_images_from_page(doc: fitz.Document, p_num: int, prefix: str, symbol_name: str, caption_hint: str = "") -> List[Dict[str, Any]]:
|
||
"""Извлекает качественные изображения со страницы и конвертирует их в PNG/JPEG."""
|
||
results = []
|
||
try:
|
||
page = doc[p_num]
|
||
imgs = page.get_images()
|
||
except Exception:
|
||
return results
|
||
|
||
if not imgs:
|
||
return results
|
||
|
||
slug = transliterate_to_slug(symbol_name)
|
||
for idx, img_info in enumerate(imgs):
|
||
try:
|
||
xref = img_info[0]
|
||
base_img = doc.extract_image(xref)
|
||
img_bytes = base_img["image"]
|
||
ext = base_img["ext"].lower()
|
||
w, h = base_img["width"], base_img["height"]
|
||
|
||
# Фильтр: игнорируем мелкие иконки, линейки, рамки
|
||
if w < 110 or h < 110 or len(img_bytes) < 12000:
|
||
continue
|
||
|
||
# Отсекаем узкие полоски
|
||
aspect = w / max(1, h)
|
||
if aspect > 5.0 or aspect < 0.2:
|
||
continue
|
||
|
||
# Конвертируем экзотические форматы (например jpx) в PNG/JPG с помощью Pillow
|
||
final_ext = "jpg" if ext in ["jpg", "jpeg"] else "png"
|
||
filename = f"{prefix}_{slug}_p{p_num+1}_{idx}_{w}x{h}.{final_ext}"
|
||
filepath = ILLUSTRATIONS_DIR / filename
|
||
|
||
if not filepath.exists():
|
||
try:
|
||
pil_img = Image.open(io.BytesIO(img_bytes))
|
||
if pil_img.mode in ("RGBA", "P") and final_ext == "jpg":
|
||
pil_img = pil_img.convert("RGB")
|
||
pil_img.save(filepath, quality=90)
|
||
except Exception:
|
||
with open(filepath, "wb") as f:
|
||
f.write(img_bytes)
|
||
|
||
clean_caption = caption_hint.strip() if caption_hint else f"Символ: {symbol_name.capitalize()}"
|
||
if len(clean_caption) > 200:
|
||
clean_caption = clean_caption[:197] + "..."
|
||
|
||
results.append({
|
||
"file": filename,
|
||
"path": f"assets/illustrations/{filename}",
|
||
"term": symbol_name,
|
||
"caption": clean_caption,
|
||
"page": p_num + 1,
|
||
"width": w,
|
||
"height": h,
|
||
"format": final_ext.upper(),
|
||
"created_at": datetime.now().isoformat()
|
||
})
|
||
except Exception:
|
||
continue
|
||
return results
|
||
|
||
|
||
def main():
|
||
print("=" * 70)
|
||
print("🚀 ГЛУБОКИЙ ПАРСИНГ БИБЛИОТЕКИ NEWSOURCES")
|
||
print("=" * 70)
|
||
|
||
ILLUSTRATIONS_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
with open(STAGING_DB_PATH, "r", encoding="utf-8") as f:
|
||
staging_db = json.load(f)
|
||
|
||
external_db = {}
|
||
if EXTERNAL_DB_PATH.exists():
|
||
with open(EXTERNAL_DB_PATH, "r", encoding="utf-8") as f:
|
||
external_db = json.load(f)
|
||
|
||
catalog = []
|
||
if CATALOG_PATH.exists():
|
||
with open(CATALOG_PATH, "r", encoding="utf-8") as f:
|
||
catalog = json.load(f)
|
||
|
||
# Индекс канонических слов 1000 знаков
|
||
core_index: Dict[str, str] = {}
|
||
for idx_str, entry in staging_db.items():
|
||
w = entry.get("word", "").lower().strip()
|
||
if w:
|
||
core_index[w] = idx_str
|
||
|
||
print(f"Канон 1000 знаков загружен. Текущих внешних символов: {len(external_db)}.")
|
||
print(f"Иллюстраций в каталоге: {len(catalog)}.")
|
||
|
||
stats = {
|
||
"cooper_entries": 0,
|
||
"karelin_entries": 0,
|
||
"kortunova_entries": 0,
|
||
"jung_entries": 0,
|
||
"baydin_entries": 0,
|
||
"goff_entries": 0,
|
||
"timofey_entries": 0,
|
||
"steiner_entries": 0,
|
||
"core_enriched": 0,
|
||
"external_added": 0,
|
||
"illustrations_added": 0
|
||
}
|
||
|
||
timestamp_str = datetime.now().isoformat()
|
||
|
||
def route_symbol(term: str, author: str, work: str, text: str, preferred_fields: List[str] = None, illustrations: List[Dict[str, Any]] = None):
|
||
"""Маршрутизирует концепт: в 1000 знаков (staging) или в oracle_external_symbols.json."""
|
||
norm = term.strip().lower()
|
||
if len(norm) < 2 or not text or len(text) < 30:
|
||
return
|
||
|
||
full_text = clean_spaces(text)
|
||
excerpt = (full_text[:400] + "...") if len(full_text) > 400 else full_text
|
||
citation = f"[{author} · «{work}»]"
|
||
quoted = f"{citation}: «{excerpt}»"
|
||
|
||
# Поиск совпадения в 1000 знаках (с учетом лемм и окончаний)
|
||
matched_idx = None
|
||
if norm in core_index:
|
||
matched_idx = core_index[norm]
|
||
elif norm.endswith("ы") and norm[:-1] in core_index:
|
||
matched_idx = core_index[norm[:-1]]
|
||
elif norm.endswith("и") and norm[:-1] in core_index:
|
||
matched_idx = core_index[norm[:-1]]
|
||
elif norm.endswith("а") and norm[:-1] in core_index:
|
||
matched_idx = core_index[norm[:-1]]
|
||
elif norm.endswith("я") and norm[:-1] in core_index:
|
||
matched_idx = core_index[norm[:-1]]
|
||
elif norm.endswith("о") and norm[:-1] in core_index:
|
||
matched_idx = core_index[norm[:-1]]
|
||
elif norm.endswith("е") and norm[:-1] in core_index:
|
||
matched_idx = core_index[norm[:-1]]
|
||
|
||
# Привязка иллюстраций
|
||
if illustrations:
|
||
for ill in illustrations:
|
||
ill["source"] = f"{author} · «{work}»"
|
||
ill["normalized"] = norm
|
||
ill["is_core_1000"] = (matched_idx is not None)
|
||
ill["core_id"] = matched_idx
|
||
if not any(x.get("file") == ill["file"] for x in catalog):
|
||
catalog.append(ill)
|
||
stats["illustrations_added"] += 1
|
||
|
||
if matched_idx:
|
||
# ОБОГАЩАЕМ КАРТОЧКУ ОРАКУЛА 999
|
||
entry = staging_db[matched_idx]
|
||
fields = entry.setdefault("fields", {})
|
||
|
||
# Ищем подходящую свободную ячейку
|
||
target_field = None
|
||
if preferred_fields:
|
||
for candidate in preferred_fields:
|
||
if not fields.get(candidate):
|
||
target_field = candidate
|
||
break
|
||
|
||
if not target_field:
|
||
# Универсальный пул ячеек
|
||
pool = [
|
||
"meta_synthesis", "semantic_core", "state_creative", "state_destructive",
|
||
"culture_antiquity_light", "culture_egypt_light", "culture_celtic_light",
|
||
"culture_nordic_light", "culture_slavic_light", "culture_india_light",
|
||
"culture_china_light", "culture_hermetic_light", "culture_sumer_light",
|
||
"culture_islam_light", "culture_kabbalah_light"
|
||
]
|
||
for p_field in pool:
|
||
if not fields.get(p_field):
|
||
target_field = p_field
|
||
break
|
||
|
||
if target_field and not fields.get(target_field):
|
||
fields[target_field] = quoted
|
||
filled, _ = count_filled_fields(entry)
|
||
entry["filled_count"] = filled
|
||
entry["is_complete"] = (filled >= 78)
|
||
stats["core_enriched"] += 1
|
||
|
||
if illustrations:
|
||
target_ills = entry.setdefault("illustrations", [])
|
||
for ill in illustrations:
|
||
if not any(x.get("file") == ill["file"] for x in target_ills):
|
||
target_ills.append(ill)
|
||
else:
|
||
# ПИШЕМ В БАЗУ ВНЕШНИХ СИМВОЛОВ
|
||
if norm not in external_db:
|
||
external_db[norm] = {
|
||
"term": term.strip(),
|
||
"normalized": norm,
|
||
"category_guess": "Внешний академический символ",
|
||
"sources": [],
|
||
"illustrations": [],
|
||
"created_at": timestamp_str,
|
||
"updated_at": timestamp_str,
|
||
}
|
||
stats["external_added"] += 1
|
||
|
||
sources_list = external_db[norm]["sources"]
|
||
if not any(s.get("author") == author and s.get("work") == work for s in sources_list):
|
||
sources_list.append({
|
||
"author": author,
|
||
"work": work,
|
||
"excerpt": full_text,
|
||
"target_field_suggestion": preferred_fields[0] if preferred_fields else "meta_synthesis"
|
||
})
|
||
external_db[norm]["updated_at"] = timestamp_str
|
||
|
||
if illustrations:
|
||
ext_ills = external_db[norm].setdefault("illustrations", [])
|
||
for ill in illustrations:
|
||
if not any(x.get("file") == ill["file"] for x in ext_ills):
|
||
ext_ills.append(ill)
|
||
|
||
# =========================================================================
|
||
# 1. ДЖ. КУПЕР — «ЭНЦИКЛОПЕДИЯ СИМВОЛОВ» (397 стр., 255 гравюр)
|
||
# =========================================================================
|
||
cooper_path = NEWSOURCES_DIR / "Kuper_Dzh_Entsiklopedia_simvolov.pdf"
|
||
if cooper_path.exists():
|
||
print("\n📖 [1/8] Обработка Дж. Купер — «Энциклопедия символов»...")
|
||
doc = fitz.open(cooper_path)
|
||
|
||
# Сканируем страницы словаря (6 - 395)
|
||
for pno in range(6, min(396, len(doc))):
|
||
raw_txt = doc[pno].get_text()
|
||
if not raw_txt or len(raw_txt) < 50:
|
||
continue
|
||
|
||
captions = re.findall(r'Рис\.\s*\d+\.\s*([^.\n]+(?:\.[^.\n]+)?)', raw_txt)
|
||
caption_hint = captions[0].strip() if captions else ""
|
||
|
||
page_imgs = extract_images_from_page(doc, pno, "cooper", f"p{pno+1}", caption_hint)
|
||
|
||
clean_txt = re.sub(r'(\w+)-\s*\n\s*(\w+)', r'\1\2', raw_txt)
|
||
clean_txt = re.sub(r'\s*\n\s*', ' ', clean_txt)
|
||
|
||
pattern = re.compile(r'(?:^|\.\s+)([А-ЯЁ][а-яёA-Za-z0-9\-\(\)«» ]{1,35})\.\s+([А-ЯЁ«].*?)(?=(?:\.\s+[А-ЯЁ][а-яёA-Za-z0-9\-\(\)«» ]{1,35}\.\s+[А-ЯЁ«])|$)')
|
||
matches = list(pattern.finditer(clean_txt))
|
||
|
||
for m in matches:
|
||
headword = m.group(1).strip()
|
||
body = m.group(2).strip()
|
||
|
||
if headword in ["Энциклопедия символов", "Иллюстрации", "Содержание", "Указатель"]:
|
||
continue
|
||
if len(body) < 35:
|
||
continue
|
||
|
||
term_clean = re.sub(r'\(.*?\)', '', headword).strip()
|
||
if not term_clean or len(term_clean) < 2:
|
||
continue
|
||
|
||
culture_segments = [
|
||
(r'(?:В Египте|У египтян)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_egypt_light", "culture_egypt_shadow"]),
|
||
(r'(?:В Греции|В Риме|У греков|У римлян)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_antiquity_light", "culture_antiquity_shadow"]),
|
||
(r'(?:У кельтов|В кельтск)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_celtic_light", "culture_celtic_shadow"]),
|
||
(r'(?:У скандинавов|В скандинавск|В германск|У тевтонов)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_nordic_light", "culture_nordic_shadow"]),
|
||
(r'(?:У славян|В славянск)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_slavic_light", "culture_slavic_shadow"]),
|
||
(r'(?:В индуизме|У индусов|В ведийск)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_india_light", "culture_india_shadow"]),
|
||
(r'(?:В Китае|У китайцев|В даосизме)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_china_light", "culture_china_shadow"]),
|
||
(r'(?:В алхимии|У алхимиков)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_hermetic_light", "culture_hermetic_shadow"]),
|
||
(r'(?:В Месопотамии|У шумеров|В Вавилоне)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_sumer_light", "culture_sumer_shadow"]),
|
||
(r'(?:В исламе|У мусульман)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_islam_light", "culture_islam_shadow"]),
|
||
(r'(?:В еврейск|В иудаизме|В каббале)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_kabbalah_light", "culture_kabbalah_shadow"]),
|
||
(r'(?:У американских индейцев|У индейцев)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_native_american_light", "culture_native_american_shadow"]),
|
||
]
|
||
|
||
route_symbol(
|
||
term=term_clean,
|
||
author="Дж. Купер",
|
||
work="Энциклопедия символов",
|
||
text=body[:500],
|
||
preferred_fields=["semantic_core", "meta_synthesis"],
|
||
illustrations=page_imgs
|
||
)
|
||
stats["cooper_entries"] += 1
|
||
|
||
for pattern_str, cult_fields in culture_segments:
|
||
cult_match = re.search(pattern_str, body, re.IGNORECASE)
|
||
if cult_match:
|
||
cult_text = cult_match.group(0).strip()
|
||
if len(cult_text) > 40:
|
||
is_shadow = any(w in cult_text.lower() for w in ["тьма", "смерт", "зло", "разруш", "демон", "ужас", "грех", "хаос"])
|
||
chosen_field = [cult_fields[1] if is_shadow else cult_fields[0]]
|
||
route_symbol(
|
||
term=term_clean,
|
||
author="Дж. Купер",
|
||
work="Энциклопедия символов",
|
||
text=cult_text,
|
||
preferred_fields=chosen_field,
|
||
illustrations=[]
|
||
)
|
||
print(f" Обработано словарных статей Купера: {stats['cooper_entries']}.")
|
||
|
||
# =========================================================================
|
||
# 2. В.В. КАРЕЛИН — «ЛАБИРИНТ. ДРАКОН. ЕДИНОРОГ…» (23 МОНОГРАФИИ)
|
||
# =========================================================================
|
||
karelin_path = NEWSOURCES_DIR / "Карелин В.В. (сост.) - Лабиринт. Дракон. Единорог… О чем говорят символы (Выставки Нового Акрополя) - 2015.pdf"
|
||
if karelin_path.exists():
|
||
print("\n📖 [2/8] Обработка В.В. Карелин — «Лабиринт. Дракон. Единорог…»...")
|
||
doc = fitz.open(karelin_path)
|
||
|
||
monographs = [
|
||
("ГОРА", 14, 21, ["semantic_core", "meta_synthesis", "state_creative"]),
|
||
("СОЛНЦЕ", 22, 29, ["solar_lunar_axis", "meta_synthesis", "state_creative"]),
|
||
("КОЛЕСО", 30, 31, ["geometric_archetype", "meta_synthesis", "state_creative"]),
|
||
("ДЕРЕВО", 32, 39, ["cosmogonic_role", "meta_synthesis", "state_creative"]),
|
||
("ЛАБИРИНТ", 40, 47, ["geometric_archetype", "meta_synthesis", "state_creative"]),
|
||
("ЗВЕЗДА", 48, 49, ["cosmogonic_role", "meta_synthesis", "state_creative"]),
|
||
("ВРАТА", 50, 55, ["spatial_orientation", "meta_synthesis", "state_creative"]),
|
||
("ЛЕСТНИЦА", 56, 63, ["spatial_orientation", "meta_synthesis", "state_creative"]),
|
||
("РЕКА", 64, 65, ["elemental_affinity", "meta_synthesis", "state_creative"]),
|
||
("ПТИЦА", 66, 73, ["archetype_category", "meta_synthesis", "state_creative"]),
|
||
("ЛАМПА", 74, 81, ["state_creative", "meta_synthesis", "will_vector"]),
|
||
("РОДНИК", 82, 89, ["elemental_affinity", "meta_synthesis", "state_creative"]),
|
||
("ДРАКОН", 90, 97, ["chthonic_aspect", "state_destructive", "meta_synthesis"]),
|
||
("КОЛОННА", 98, 99, ["spatial_orientation", "meta_synthesis", "state_creative"]),
|
||
("ЕДИНОРОГ", 100, 107, ["archetype_category", "meta_synthesis", "state_creative"]),
|
||
("МЕЧ", 108, 115, ["will_vector", "meta_synthesis", "state_creative"]),
|
||
("СЕРДЦЕ", 116, 125, ["philosophical_essence", "meta_synthesis", "state_creative"]),
|
||
("МОСТ", 126, 133, ["spatial_orientation", "meta_synthesis", "state_creative"]),
|
||
("ЛИРА", 134, 141, ["state_creative", "meta_synthesis", "will_vector"]),
|
||
("ЩИТ", 142, 149, ["will_vector", "meta_synthesis", "state_creative"]),
|
||
("МОЛОТ", 150, 151, ["will_vector", "meta_synthesis", "state_creative"]),
|
||
("ТОПОР", 152, 153, ["will_vector", "meta_synthesis", "state_destructive"]),
|
||
("КЛЮЧ", 154, 155, ["meta_synthesis", "state_creative", "will_vector"]),
|
||
]
|
||
|
||
for sym_name, start_p, end_p, pref_fields in monographs:
|
||
chapter_text = []
|
||
chapter_imgs = []
|
||
for p in range(start_p - 1, min(end_p, len(doc))):
|
||
chapter_text.append(doc[p].get_text())
|
||
imgs = extract_images_from_page(doc, p, "karelin", sym_name, f"Монография: {sym_name}")
|
||
chapter_imgs.extend(imgs)
|
||
|
||
full_chapter_txt = clean_spaces(" ".join(chapter_text))
|
||
if len(full_chapter_txt) > 100:
|
||
route_symbol(
|
||
term=sym_name,
|
||
author="В.В. Карелин",
|
||
work="Лабиринт. Дракон. Единорог… О чем говорят символы",
|
||
text=full_chapter_txt[:1200],
|
||
preferred_fields=pref_fields,
|
||
illustrations=chapter_imgs
|
||
)
|
||
stats["karelin_entries"] += 1
|
||
print(f" Обработано монографий Карелина: {stats['karelin_entries']}.")
|
||
|
||
# =========================================================================
|
||
# 3. Н.Д. КОРТУНОВА — «ЗНАКИ, СИМВОЛЫ И АЛЛЕГОРИИ В ИСКУССТВЕ»
|
||
# =========================================================================
|
||
kortunova_path = NEWSOURCES_DIR / "Кортунова Н. Д. - Знаки, символы и аллегории в искусстве (Искусство понимать) - 2023.pdf"
|
||
if kortunova_path.exists():
|
||
print("\n📖 [3/8] Обработка Н.Д. Кортунова — «Знаки, символы и аллегории в искусстве»...")
|
||
doc = fitz.open(kortunova_path)
|
||
|
||
for pno in range(15, min(185, len(doc))):
|
||
raw_txt = doc[pno].get_text()
|
||
if not raw_txt or len(raw_txt) < 80:
|
||
continue
|
||
|
||
lines = [l.strip() for l in raw_txt.split('\n') if l.strip()]
|
||
candidate_term = None
|
||
for l in lines[:4]:
|
||
if len(l) < 25 and re.match(r'^[А-ЯЁ][а-яё\- ]+$', l):
|
||
if l not in ["Глава I", "Глава II", "Глава III", "Глава IV", "Флора", "Фауна", "Мир предметов", "Аллегории", "Знаки, символы и аллегории в искусстве"]:
|
||
candidate_term = l
|
||
break
|
||
|
||
if candidate_term:
|
||
clean_body = clean_spaces(raw_txt)
|
||
page_imgs = extract_images_from_page(doc, pno, "kortunova", candidate_term, f"Шедевр искусства: {candidate_term}")
|
||
route_symbol(
|
||
term=candidate_term,
|
||
author="Н.Д. Кортунова",
|
||
work="Знаки, символы и аллегории в искусстве",
|
||
text=clean_body[:800],
|
||
preferred_fields=["culture_antiquity_light", "culture_hermetic_light", "semantic_core"],
|
||
illustrations=page_imgs
|
||
)
|
||
stats["kortunova_entries"] += 1
|
||
print(f" Обработано разделов Кортуновой: {stats['kortunova_entries']}.")
|
||
|
||
# =========================================================================
|
||
# 4. К.Г. ЮНГ — ТРИ ТОМА АРХЕТИПИЧЕСКОЙ СИМВОЛИКИ
|
||
# =========================================================================
|
||
jung_books = [
|
||
("Символы трансформации", NEWSOURCES_DIR / "Карл Густав Юнг - Символы трансформации (Психологические технологии) - 2020.pdf"),
|
||
("Символы сновидений", NEWSOURCES_DIR / "Карл Густав Юнг - Символы сновидений в процессе индивидуации - 2021.pdf"),
|
||
("Человек и его символы", NEWSOURCES_DIR / "Юнг К. Г. - Человек и его символы - 2020.pdf"),
|
||
]
|
||
|
||
jung_archetypes = [
|
||
("ГЕРОЙ", ["state_creative", "will_vector", "meta_synthesis"]),
|
||
("МАТЬ", ["cosmogonic_role", "meta_synthesis", "state_creative"]),
|
||
("ЗМЕЙ", ["chthonic_aspect", "state_destructive", "meta_synthesis"]),
|
||
("ДРАКОН", ["chthonic_aspect", "state_destructive", "meta_synthesis"]),
|
||
("ДРЕВО", ["cosmogonic_role", "meta_synthesis", "state_creative"]),
|
||
("ВОДА", ["elemental_affinity", "state_creative", "meta_synthesis"]),
|
||
("ОГОНЬ", ["elemental_affinity", "state_creative", "meta_synthesis"]),
|
||
("СОЛНЦЕ", ["solar_lunar_axis", "state_creative", "meta_synthesis"]),
|
||
("ЖЕРТВА", ["state_destructive", "will_vector", "meta_synthesis"]),
|
||
("ТЕНЬ", ["state_destructive", "shadow_warning", "meta_synthesis"]),
|
||
("ЗОЛОТО", ["culture_hermetic_light", "meta_synthesis", "state_creative"]),
|
||
("КАМЕНЬ", ["culture_hermetic_light", "meta_synthesis", "state_creative"]),
|
||
("МАНДАЛА", ["geometric_archetype", "meta_synthesis", "state_creative"]),
|
||
("САМОСТЬ", ["meta_synthesis", "state_creative", "semantic_core"]),
|
||
("УРОБОРОС", ["geometric_archetype", "meta_synthesis", "chthonic_aspect"]),
|
||
("ЛАБИРИНТ", ["geometric_archetype", "meta_synthesis", "state_creative"]),
|
||
("КОНЬ", ["archetype_category", "state_creative", "meta_synthesis"]),
|
||
("РЫБА", ["archetype_category", "state_creative", "meta_synthesis"]),
|
||
("ПТИЦА", ["archetype_category", "state_creative", "meta_synthesis"]),
|
||
]
|
||
|
||
print("\n📖 [4/8] Обработка наследия К.Г. Юнга...")
|
||
for j_title, j_file in jung_books:
|
||
if not j_file.exists():
|
||
continue
|
||
doc = fitz.open(j_file)
|
||
for pno in range(min(500, len(doc))):
|
||
page_text = doc[pno].get_text()
|
||
if not page_text or len(page_text) < 100:
|
||
continue
|
||
|
||
for arch, p_fields in jung_archetypes:
|
||
pattern = rf'\b{arch.capitalize()}[а-яё]*\b[^\.\n]*?(?:архетип|символ|бессознательн|либидо|трансформац|сновиден)[^\.\n]*?\.'
|
||
m = re.search(pattern, page_text, re.IGNORECASE)
|
||
if m:
|
||
snippet = page_text[max(0, m.start() - 100):min(len(page_text), m.end() + 350)]
|
||
clean_snip = clean_spaces(snippet)
|
||
if len(clean_snip) > 70:
|
||
route_symbol(
|
||
term=arch,
|
||
author="К.Г. Юнг",
|
||
work=j_title,
|
||
text=clean_snip,
|
||
preferred_fields=p_fields,
|
||
illustrations=[]
|
||
)
|
||
stats["jung_entries"] += 1
|
||
break
|
||
print(f" Извлечено цитат Юнга: {stats['jung_entries']}.")
|
||
|
||
# =========================================================================
|
||
# 5. В.В. БАЙДИН — «АРХЕТИПЫ И СИМВОЛЫ РУССКОЙ КУЛЬТУРЫ» (FB2)
|
||
# =========================================================================
|
||
baydin_path = NEWSOURCES_DIR / "Байдин В.В. - Архетипы и символы русской культуры. От архаики до современности - 2021" / "Байдин В.В. - Архетипы и символы русской культуры - 2021.fb2"
|
||
if baydin_path.exists():
|
||
print("\n📖 [5/8] Обработка В.В. Байдин — «Архетипы и символы русской культуры»...")
|
||
try:
|
||
with open(baydin_path, "r", encoding="utf-8", errors="ignore") as f:
|
||
fb2_content = f.read()
|
||
|
||
slavic_archetypes = [
|
||
("СОЛНЦЕ", ["culture_slavic_light", "script_slavic"]),
|
||
("ОГОНЬ", ["culture_slavic_light", "script_slavic"]),
|
||
("ЗЕМЛЯ", ["culture_slavic_light", "script_slavic"]),
|
||
("ВОДА", ["culture_slavic_light", "script_slavic"]),
|
||
("ДЕРЕВО", ["culture_slavic_light", "script_slavic"]),
|
||
("ДУБ", ["culture_slavic_light", "script_slavic"]),
|
||
("КОНЬ", ["culture_slavic_light", "script_slavic"]),
|
||
("ПТИЦА", ["culture_slavic_light", "script_slavic"]),
|
||
("ХРАМ", ["culture_slavic_light", "script_slavic"]),
|
||
("КРЕСТ", ["culture_slavic_light", "script_slavic"]),
|
||
("КОЛОВРАТ", ["culture_slavic_light", "script_slavic"]),
|
||
("ЗМЕЙ", ["culture_slavic_shadow", "script_slavic"]),
|
||
]
|
||
|
||
for s_term, s_fields in slavic_archetypes:
|
||
pattern = rf'<p>[^<]*?\b{s_term.capitalize()}[а-яё]*\b[^<]*?(?:славян|святилищ|архаик|курган|культ|древнерус)[^<]*?</p>'
|
||
matches = re.findall(pattern, fb2_content, re.IGNORECASE)
|
||
for raw_p in matches[:3]:
|
||
clean_p = clean_spaces(re.sub(r'<[^>]+>', '', raw_p))
|
||
if len(clean_p) > 50:
|
||
route_symbol(
|
||
term=s_term,
|
||
author="В.В. Байдин",
|
||
work="Архетипы и символы русской культуры",
|
||
text=clean_p,
|
||
preferred_fields=s_fields,
|
||
illustrations=[]
|
||
)
|
||
stats["baydin_entries"] += 1
|
||
print(f" Извлечено славянских архетипов Байдина: {stats['baydin_entries']}.")
|
||
except Exception as e:
|
||
print(f" Ошибка чтения Байдина: {e}")
|
||
|
||
# =========================================================================
|
||
# 6. Б. ГОФФ — «SYMBOLS OF PREHISTORIC MESOPOTAMIA» (498 стр.)
|
||
# =========================================================================
|
||
goff_path = NEWSOURCES_DIR / "_Goff B. Symbols of prehistoric Mesopotamia (1963).pdf"
|
||
if goff_path.exists():
|
||
print("\n📖 [6/8] Обработка Б. Гофф — «Symbols of Prehistoric Mesopotamia»...")
|
||
doc = fitz.open(goff_path)
|
||
|
||
meso_motifs = [
|
||
("РОЗЕТКА", "rosette", ["culture_sumer_light", "script_sumer_akkad"]),
|
||
("БЫК", "bull", ["culture_sumer_light", "script_sumer_akkad"]),
|
||
("ЗМЕЯ", "serpent", ["culture_sumer_light", "script_sumer_akkad"]),
|
||
("ДРЕВО", "tree", ["culture_sumer_light", "script_sumer_akkad"]),
|
||
("СОСУД", "vase", ["culture_sumer_light", "script_sumer_akkad"]),
|
||
("ЛЕВ", "lion", ["culture_sumer_light", "script_sumer_akkad"]),
|
||
("РОГА", "horns", ["culture_sumer_light", "script_sumer_akkad"]),
|
||
]
|
||
|
||
for pno in range(30, min(300, len(doc))):
|
||
raw_text = doc[pno].get_text()
|
||
if not raw_text or len(raw_text) < 100:
|
||
continue
|
||
|
||
for ru_term, en_term, s_fields in meso_motifs:
|
||
if en_term in raw_text.lower():
|
||
sentences = re.findall(rf'[^.!?]*\b{en_term}\b[^.!?]*[.!?]', raw_text, re.IGNORECASE)
|
||
if sentences:
|
||
eng_text = clean_spaces(" ".join(sentences[:2]))
|
||
page_imgs = extract_images_from_page(doc, pno, "goff", ru_term, f"Prehistoric Mesopotamian: {en_term.capitalize()}")
|
||
route_symbol(
|
||
term=ru_term,
|
||
author="Б. Гофф (Beatrice Goff)",
|
||
work="Symbols of Prehistoric Mesopotamia",
|
||
text=eng_text,
|
||
preferred_fields=s_fields,
|
||
illustrations=page_imgs
|
||
)
|
||
stats["goff_entries"] += 1
|
||
break
|
||
print(f" Извлечено месопотамских мотивов Гофф: {stats['goff_entries']}.")
|
||
|
||
# =========================================================================
|
||
# 7. ТИМОФЕЙ ИЗ ГАЗЫ — «ТИГРИЦА И ГРИФОН» (БЕСТИАРИЙ)
|
||
# =========================================================================
|
||
timofey_path = NEWSOURCES_DIR / "Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002" / "Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002.pdf"
|
||
if timofey_path.exists():
|
||
print("\n📖 [7/8] Обработка Тимофей из Газы — «Тигрица и грифон»...")
|
||
doc = fitz.open(timofey_path)
|
||
|
||
bestiary_animals = [
|
||
("ЛЕВ", ["culture_antiquity_light", "archetype_category"]),
|
||
("ОРЕЛ", ["culture_antiquity_light", "archetype_category"]),
|
||
("ОЛЕНЬ", ["culture_antiquity_light", "archetype_category"]),
|
||
("ДЕЛЬФИН", ["culture_antiquity_light", "archetype_category"]),
|
||
("СЛОН", ["culture_antiquity_light", "archetype_category"]),
|
||
("СКОРПИОН", ["chthonic_aspect", "culture_antiquity_shadow"]),
|
||
("ПАНТЕРА", ["culture_antiquity_light", "archetype_category"]),
|
||
("ГРИФОН", ["archetype_category", "meta_synthesis"]),
|
||
("ВАСИЛИСК", ["chthonic_aspect", "state_destructive"]),
|
||
("ФЕНИКС", ["state_creative", "meta_synthesis"]),
|
||
("ВОРОН", ["culture_antiquity_light", "archetype_category"]),
|
||
("СОВА", ["culture_antiquity_light", "archetype_category"]),
|
||
("ПАУК", ["chthonic_aspect", "culture_antiquity_shadow"]),
|
||
("ПЧЕЛА", ["state_creative", "culture_antiquity_light"]),
|
||
]
|
||
|
||
for pno in range(20, min(350, len(doc))):
|
||
raw_text = doc[pno].get_text()
|
||
if not raw_text or len(raw_text) < 100:
|
||
continue
|
||
|
||
for animal, a_fields in bestiary_animals:
|
||
m = re.search(rf'\b{animal.capitalize()}[а-яё]*\b[^\.\n]*?(?:сакральн|символ|природ|звер|почитан|божеств)[^\.\n]*?\.', raw_text, re.IGNORECASE)
|
||
if m:
|
||
snippet = clean_spaces(raw_text[max(0, m.start() - 80):min(len(raw_text), m.end() + 300)])
|
||
page_imgs = extract_images_from_page(doc, pno, "timofey", animal, f"Бестиарий: {animal}")
|
||
route_symbol(
|
||
term=animal,
|
||
author="Тимофей из Газы",
|
||
work="Тигрица и грифон. Сакральные символы животного мира",
|
||
text=snippet,
|
||
preferred_fields=a_fields,
|
||
illustrations=page_imgs
|
||
)
|
||
stats["timofey_entries"] += 1
|
||
break
|
||
print(f" Извлечено статей бестиария Тимофея: {stats['timofey_entries']}.")
|
||
|
||
# =========================================================================
|
||
# 8. Р. ШТАЙНЕР — «ОККУЛЬТНЫЕ ЗНАКИ И СИМВОЛЫ» (FB2)
|
||
# =========================================================================
|
||
steiner_path = NEWSOURCES_DIR / "Rudolf_Shtajner__Okkultnye_znaki_i_simvoly_Lekcii_14.fb2"
|
||
if steiner_path.exists():
|
||
print("\n📖 [8/8] Обработка Р. Штайнер — «Оккультные знаки и символы»...")
|
||
try:
|
||
with open(steiner_path, "r", encoding="utf-8", errors="ignore") as f:
|
||
fb2_steiner = f.read()
|
||
|
||
steiner_symbols = [
|
||
("ПЕНТАГРАММА", ["geometric_archetype", "meta_synthesis"]),
|
||
("ГЕКСАГРАММА", ["geometric_archetype", "meta_synthesis"]),
|
||
("СВАСТИКА", ["geometric_archetype", "meta_synthesis"]),
|
||
("КРЕСТ", ["meta_synthesis", "culture_hermetic_light"]),
|
||
("РОЗА", ["meta_synthesis", "culture_hermetic_light"]),
|
||
]
|
||
|
||
for st_term, st_fields in steiner_symbols:
|
||
pattern = rf'<p>[^<]*?\b{st_term.capitalize()}[а-яё]*\b[^<]*?(?:оккультн|духовн|знак|символ|микрокосм|макрокосм)[^<]*?</p>'
|
||
matches = re.findall(pattern, fb2_steiner, re.IGNORECASE)
|
||
for raw_p in matches[:3]:
|
||
clean_p = clean_spaces(re.sub(r'<[^>]+>', '', raw_p))
|
||
if len(clean_p) > 60:
|
||
route_symbol(
|
||
term=st_term,
|
||
author="Р. Штайнер",
|
||
work="Оккультные знаки и символы",
|
||
text=clean_p,
|
||
preferred_fields=st_fields,
|
||
illustrations=[]
|
||
)
|
||
stats["steiner_entries"] += 1
|
||
print(f" Извлечено символов Штайнера: {stats['steiner_entries']}.")
|
||
except Exception as e:
|
||
print(f" Ошибка чтения Штайнера: {e}")
|
||
|
||
# =========================================================================
|
||
# СОХРАНЕНИЕ РЕЗУЛЬТАТОВ
|
||
# =========================================================================
|
||
print("\n💾 Сохранение баз данных...")
|
||
with open(STAGING_DB_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(staging_db, f, ensure_ascii=False, indent=2)
|
||
print(f"✅ Staging база сохранена: {STAGING_DB_PATH}")
|
||
|
||
MAIN_DB_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_passports.json"
|
||
with open(MAIN_DB_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(staging_db, f, ensure_ascii=False, indent=2)
|
||
print(f"✅ Основная база синхронизирована: {MAIN_DB_PATH}")
|
||
|
||
with open(EXTERNAL_DB_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(external_db, f, ensure_ascii=False, indent=2)
|
||
print(f"✅ База внешних символов сохранена: {EXTERNAL_DB_PATH} ({len(external_db)} терминов)")
|
||
|
||
with open(ENRICHER_EXTERNAL_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(external_db, f, ensure_ascii=False, indent=2)
|
||
print(f"✅ Копия в oracle_enricher сохранена: {ENRICHER_EXTERNAL_PATH}")
|
||
|
||
with open(CATALOG_PATH, "w", encoding="utf-8") as f:
|
||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||
print(f"✅ Каталог иллюстраций обновлен: {CATALOG_PATH} ({len(catalog)} изображений)")
|
||
|
||
print("\n" + "=" * 70)
|
||
print("📊 ИТОГОВЫЙ ОТЧЕТ ИНГЕСТИИ:")
|
||
print(f" • Обогащено ячеек Оракула 999: +{stats['core_enriched']}")
|
||
print(f" • Добавлено внешних символов: +{stats['external_added']}")
|
||
print(f" • Всего внешних символов в базе: {len(external_db)}")
|
||
print(f" • Новых иллюстраций извлечено: +{stats['illustrations_added']}")
|
||
print(f" • Всего иллюстраций в каталоге: {len(catalog)}")
|
||
print("=" * 70)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|