#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
ingest_secondary_deep_corpus.py
===============================
Глубокий парсер академических источников из /home/alexives/Desktop/newsources:
1. Дж. Купер — «Энциклопедия символов» (1995, 397 стр.) + 250+ гравюр
2. В.В. Карелин — «Лабиринт. Дракон. Единорог… О чем говорят символы» (2015, 163 стр.) + иллюстрации
3. Н.Д. Кортунова — «Знаки, символы и аллегории в искусстве» (2023, 193 стр.) + шедевры живописи
4. К.Г. Юнг — 3 тома: «Символы трансформации», «Символы сновидений», «Человек и его символы»
5. В.В. Байдин — «Архетипы и символы русской культуры» (2021, FB2)
6. Б. Гофф — «Symbols of Prehistoric Mesopotamia» (1963, 498 стр.)
7. Тимофей из Газы — «Тигрица и грифон. Сакральные символы животного мира» (2002)
8. Р. Штайнер — «Оккультные знаки и символы» (FB2)
ПРАВИЛА:
- Приоритет: 1000 знаков Оракула (000–999). Каждая культура ложится в соответствующую ячейку Раздела V.
- Все неучтенное: попадает в oracle_external_symbols.json.
- Картинки: обязательны, сохраняются в assets/illustrations/ и привязываются к объектам.
- Цитаты: строго аутентичные, в формате [Автор · «Книга»]: «...».
"""
import os
import sys
import json
import re
import io
from pathlib import Path
from typing import Dict, Any, List, Optional, Tuple, Set
from datetime import datetime
from PIL import Image
PROJECT_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
import fitz
from oracle_enricher.enricher.passport_schema import count_filled_fields
NEWSOURCES_DIR = Path("/home/alexives/Desktop/newsources")
ILLUSTRATIONS_DIR = PROJECT_ROOT / "assets" / "illustrations"
CATALOG_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_illustrations_catalog.json"
STAGING_DB_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_passports_staging.json"
EXTERNAL_DB_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_external_symbols.json"
ENRICHER_EXTERNAL_PATH = PROJECT_ROOT / "oracle_enricher" / "data" / "oracle_external_symbols.json"
def clean_spaces(text: str) -> str:
text = re.sub(r'(\w+)-\s*\n\s*(\w+)', r'\1\2', text)
return re.sub(r'\s+', ' ', text).strip()
def transliterate_to_slug(text: str) -> str:
ru_to_en = {
'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo',
'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm',
'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u',
'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'shch',
'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya'
}
slug = []
for c in text.lower():
if c in ru_to_en:
slug.append(ru_to_en[c])
elif c.isalnum():
slug.append(c)
elif c in [' ', '-', '_']:
slug.append('_')
res = re.sub(r'_+', '_', ''.join(slug)).strip('_')
return res[:30] if res else 'symbol'
def extract_images_from_page(doc: fitz.Document, p_num: int, prefix: str, symbol_name: str, caption_hint: str = "") -> List[Dict[str, Any]]:
"""Извлекает качественные изображения со страницы и конвертирует их в PNG/JPEG."""
results = []
try:
page = doc[p_num]
imgs = page.get_images()
except Exception:
return results
if not imgs:
return results
slug = transliterate_to_slug(symbol_name)
for idx, img_info in enumerate(imgs):
try:
xref = img_info[0]
base_img = doc.extract_image(xref)
img_bytes = base_img["image"]
ext = base_img["ext"].lower()
w, h = base_img["width"], base_img["height"]
# Фильтр: игнорируем мелкие иконки, линейки, рамки
if w < 110 or h < 110 or len(img_bytes) < 12000:
continue
# Отсекаем узкие полоски
aspect = w / max(1, h)
if aspect > 5.0 or aspect < 0.2:
continue
# Конвертируем экзотические форматы (например jpx) в PNG/JPG с помощью Pillow
final_ext = "jpg" if ext in ["jpg", "jpeg"] else "png"
filename = f"{prefix}_{slug}_p{p_num+1}_{idx}_{w}x{h}.{final_ext}"
filepath = ILLUSTRATIONS_DIR / filename
if not filepath.exists():
try:
pil_img = Image.open(io.BytesIO(img_bytes))
if pil_img.mode in ("RGBA", "P") and final_ext == "jpg":
pil_img = pil_img.convert("RGB")
pil_img.save(filepath, quality=90)
except Exception:
with open(filepath, "wb") as f:
f.write(img_bytes)
clean_caption = caption_hint.strip() if caption_hint else f"Символ: {symbol_name.capitalize()}"
if len(clean_caption) > 200:
clean_caption = clean_caption[:197] + "..."
results.append({
"file": filename,
"path": f"assets/illustrations/{filename}",
"term": symbol_name,
"caption": clean_caption,
"page": p_num + 1,
"width": w,
"height": h,
"format": final_ext.upper(),
"created_at": datetime.now().isoformat()
})
except Exception:
continue
return results
def main():
print("=" * 70)
print("🚀 ГЛУБОКИЙ ПАРСИНГ БИБЛИОТЕКИ NEWSOURCES")
print("=" * 70)
ILLUSTRATIONS_DIR.mkdir(parents=True, exist_ok=True)
with open(STAGING_DB_PATH, "r", encoding="utf-8") as f:
staging_db = json.load(f)
external_db = {}
if EXTERNAL_DB_PATH.exists():
with open(EXTERNAL_DB_PATH, "r", encoding="utf-8") as f:
external_db = json.load(f)
catalog = []
if CATALOG_PATH.exists():
with open(CATALOG_PATH, "r", encoding="utf-8") as f:
catalog = json.load(f)
# Индекс канонических слов 1000 знаков
core_index: Dict[str, str] = {}
for idx_str, entry in staging_db.items():
w = entry.get("word", "").lower().strip()
if w:
core_index[w] = idx_str
print(f"Канон 1000 знаков загружен. Текущих внешних символов: {len(external_db)}.")
print(f"Иллюстраций в каталоге: {len(catalog)}.")
stats = {
"cooper_entries": 0,
"karelin_entries": 0,
"kortunova_entries": 0,
"jung_entries": 0,
"baydin_entries": 0,
"goff_entries": 0,
"timofey_entries": 0,
"steiner_entries": 0,
"core_enriched": 0,
"external_added": 0,
"illustrations_added": 0
}
timestamp_str = datetime.now().isoformat()
def route_symbol(term: str, author: str, work: str, text: str, preferred_fields: List[str] = None, illustrations: List[Dict[str, Any]] = None):
"""Маршрутизирует концепт: в 1000 знаков (staging) или в oracle_external_symbols.json."""
norm = term.strip().lower()
if len(norm) < 2 or not text or len(text) < 30:
return
full_text = clean_spaces(text)
excerpt = (full_text[:400] + "...") if len(full_text) > 400 else full_text
citation = f"[{author} · «{work}»]"
quoted = f"{citation}: «{excerpt}»"
# Поиск совпадения в 1000 знаках (с учетом лемм и окончаний)
matched_idx = None
if norm in core_index:
matched_idx = core_index[norm]
elif norm.endswith("ы") and norm[:-1] in core_index:
matched_idx = core_index[norm[:-1]]
elif norm.endswith("и") and norm[:-1] in core_index:
matched_idx = core_index[norm[:-1]]
elif norm.endswith("а") and norm[:-1] in core_index:
matched_idx = core_index[norm[:-1]]
elif norm.endswith("я") and norm[:-1] in core_index:
matched_idx = core_index[norm[:-1]]
elif norm.endswith("о") and norm[:-1] in core_index:
matched_idx = core_index[norm[:-1]]
elif norm.endswith("е") and norm[:-1] in core_index:
matched_idx = core_index[norm[:-1]]
# Привязка иллюстраций
if illustrations:
for ill in illustrations:
ill["source"] = f"{author} · «{work}»"
ill["normalized"] = norm
ill["is_core_1000"] = (matched_idx is not None)
ill["core_id"] = matched_idx
if not any(x.get("file") == ill["file"] for x in catalog):
catalog.append(ill)
stats["illustrations_added"] += 1
if matched_idx:
# ОБОГАЩАЕМ КАРТОЧКУ ОРАКУЛА 999
entry = staging_db[matched_idx]
fields = entry.setdefault("fields", {})
# Ищем подходящую свободную ячейку
target_field = None
if preferred_fields:
for candidate in preferred_fields:
if not fields.get(candidate):
target_field = candidate
break
if not target_field:
# Универсальный пул ячеек
pool = [
"meta_synthesis", "semantic_core", "state_creative", "state_destructive",
"culture_antiquity_light", "culture_egypt_light", "culture_celtic_light",
"culture_nordic_light", "culture_slavic_light", "culture_india_light",
"culture_china_light", "culture_hermetic_light", "culture_sumer_light",
"culture_islam_light", "culture_kabbalah_light"
]
for p_field in pool:
if not fields.get(p_field):
target_field = p_field
break
if target_field and not fields.get(target_field):
fields[target_field] = quoted
filled, _ = count_filled_fields(entry)
entry["filled_count"] = filled
entry["is_complete"] = (filled >= 78)
stats["core_enriched"] += 1
if illustrations:
target_ills = entry.setdefault("illustrations", [])
for ill in illustrations:
if not any(x.get("file") == ill["file"] for x in target_ills):
target_ills.append(ill)
else:
# ПИШЕМ В БАЗУ ВНЕШНИХ СИМВОЛОВ
if norm not in external_db:
external_db[norm] = {
"term": term.strip(),
"normalized": norm,
"category_guess": "Внешний академический символ",
"sources": [],
"illustrations": [],
"created_at": timestamp_str,
"updated_at": timestamp_str,
}
stats["external_added"] += 1
sources_list = external_db[norm]["sources"]
if not any(s.get("author") == author and s.get("work") == work for s in sources_list):
sources_list.append({
"author": author,
"work": work,
"excerpt": full_text,
"target_field_suggestion": preferred_fields[0] if preferred_fields else "meta_synthesis"
})
external_db[norm]["updated_at"] = timestamp_str
if illustrations:
ext_ills = external_db[norm].setdefault("illustrations", [])
for ill in illustrations:
if not any(x.get("file") == ill["file"] for x in ext_ills):
ext_ills.append(ill)
# =========================================================================
# 1. ДЖ. КУПЕР — «ЭНЦИКЛОПЕДИЯ СИМВОЛОВ» (397 стр., 255 гравюр)
# =========================================================================
cooper_path = NEWSOURCES_DIR / "Kuper_Dzh_Entsiklopedia_simvolov.pdf"
if cooper_path.exists():
print("\n📖 [1/8] Обработка Дж. Купер — «Энциклопедия символов»...")
doc = fitz.open(cooper_path)
# Сканируем страницы словаря (6 - 395)
for pno in range(6, min(396, len(doc))):
raw_txt = doc[pno].get_text()
if not raw_txt or len(raw_txt) < 50:
continue
captions = re.findall(r'Рис\.\s*\d+\.\s*([^.\n]+(?:\.[^.\n]+)?)', raw_txt)
caption_hint = captions[0].strip() if captions else ""
page_imgs = extract_images_from_page(doc, pno, "cooper", f"p{pno+1}", caption_hint)
clean_txt = re.sub(r'(\w+)-\s*\n\s*(\w+)', r'\1\2', raw_txt)
clean_txt = re.sub(r'\s*\n\s*', ' ', clean_txt)
pattern = re.compile(r'(?:^|\.\s+)([А-ЯЁ][а-яёA-Za-z0-9\-\(\)«» ]{1,35})\.\s+([А-ЯЁ«].*?)(?=(?:\.\s+[А-ЯЁ][а-яёA-Za-z0-9\-\(\)«» ]{1,35}\.\s+[А-ЯЁ«])|$)')
matches = list(pattern.finditer(clean_txt))
for m in matches:
headword = m.group(1).strip()
body = m.group(2).strip()
if headword in ["Энциклопедия символов", "Иллюстрации", "Содержание", "Указатель"]:
continue
if len(body) < 35:
continue
term_clean = re.sub(r'\(.*?\)', '', headword).strip()
if not term_clean or len(term_clean) < 2:
continue
culture_segments = [
(r'(?:В Египте|У египтян)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_egypt_light", "culture_egypt_shadow"]),
(r'(?:В Греции|В Риме|У греков|У римлян)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_antiquity_light", "culture_antiquity_shadow"]),
(r'(?:У кельтов|В кельтск)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_celtic_light", "culture_celtic_shadow"]),
(r'(?:У скандинавов|В скандинавск|В германск|У тевтонов)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_nordic_light", "culture_nordic_shadow"]),
(r'(?:У славян|В славянск)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_slavic_light", "culture_slavic_shadow"]),
(r'(?:В индуизме|У индусов|В ведийск)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_india_light", "culture_india_shadow"]),
(r'(?:В Китае|У китайцев|В даосизме)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_china_light", "culture_china_shadow"]),
(r'(?:В алхимии|У алхимиков)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_hermetic_light", "culture_hermetic_shadow"]),
(r'(?:В Месопотамии|У шумеров|В Вавилоне)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_sumer_light", "culture_sumer_shadow"]),
(r'(?:В исламе|У мусульман)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_islam_light", "culture_islam_shadow"]),
(r'(?:В еврейск|В иудаизме|В каббале)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_kabbalah_light", "culture_kabbalah_shadow"]),
(r'(?:У американских индейцев|У индейцев)[^.]*\.(.*?)(?=(?:В [А-Я]|У [А-Я]|$))', ["culture_native_american_light", "culture_native_american_shadow"]),
]
route_symbol(
term=term_clean,
author="Дж. Купер",
work="Энциклопедия символов",
text=body[:500],
preferred_fields=["semantic_core", "meta_synthesis"],
illustrations=page_imgs
)
stats["cooper_entries"] += 1
for pattern_str, cult_fields in culture_segments:
cult_match = re.search(pattern_str, body, re.IGNORECASE)
if cult_match:
cult_text = cult_match.group(0).strip()
if len(cult_text) > 40:
is_shadow = any(w in cult_text.lower() for w in ["тьма", "смерт", "зло", "разруш", "демон", "ужас", "грех", "хаос"])
chosen_field = [cult_fields[1] if is_shadow else cult_fields[0]]
route_symbol(
term=term_clean,
author="Дж. Купер",
work="Энциклопедия символов",
text=cult_text,
preferred_fields=chosen_field,
illustrations=[]
)
print(f" Обработано словарных статей Купера: {stats['cooper_entries']}.")
# =========================================================================
# 2. В.В. КАРЕЛИН — «ЛАБИРИНТ. ДРАКОН. ЕДИНОРОГ…» (23 МОНОГРАФИИ)
# =========================================================================
karelin_path = NEWSOURCES_DIR / "Карелин В.В. (сост.) - Лабиринт. Дракон. Единорог… О чем говорят символы (Выставки Нового Акрополя) - 2015.pdf"
if karelin_path.exists():
print("\n📖 [2/8] Обработка В.В. Карелин — «Лабиринт. Дракон. Единорог…»...")
doc = fitz.open(karelin_path)
monographs = [
("ГОРА", 14, 21, ["semantic_core", "meta_synthesis", "state_creative"]),
("СОЛНЦЕ", 22, 29, ["solar_lunar_axis", "meta_synthesis", "state_creative"]),
("КОЛЕСО", 30, 31, ["geometric_archetype", "meta_synthesis", "state_creative"]),
("ДЕРЕВО", 32, 39, ["cosmogonic_role", "meta_synthesis", "state_creative"]),
("ЛАБИРИНТ", 40, 47, ["geometric_archetype", "meta_synthesis", "state_creative"]),
("ЗВЕЗДА", 48, 49, ["cosmogonic_role", "meta_synthesis", "state_creative"]),
("ВРАТА", 50, 55, ["spatial_orientation", "meta_synthesis", "state_creative"]),
("ЛЕСТНИЦА", 56, 63, ["spatial_orientation", "meta_synthesis", "state_creative"]),
("РЕКА", 64, 65, ["elemental_affinity", "meta_synthesis", "state_creative"]),
("ПТИЦА", 66, 73, ["archetype_category", "meta_synthesis", "state_creative"]),
("ЛАМПА", 74, 81, ["state_creative", "meta_synthesis", "will_vector"]),
("РОДНИК", 82, 89, ["elemental_affinity", "meta_synthesis", "state_creative"]),
("ДРАКОН", 90, 97, ["chthonic_aspect", "state_destructive", "meta_synthesis"]),
("КОЛОННА", 98, 99, ["spatial_orientation", "meta_synthesis", "state_creative"]),
("ЕДИНОРОГ", 100, 107, ["archetype_category", "meta_synthesis", "state_creative"]),
("МЕЧ", 108, 115, ["will_vector", "meta_synthesis", "state_creative"]),
("СЕРДЦЕ", 116, 125, ["philosophical_essence", "meta_synthesis", "state_creative"]),
("МОСТ", 126, 133, ["spatial_orientation", "meta_synthesis", "state_creative"]),
("ЛИРА", 134, 141, ["state_creative", "meta_synthesis", "will_vector"]),
("ЩИТ", 142, 149, ["will_vector", "meta_synthesis", "state_creative"]),
("МОЛОТ", 150, 151, ["will_vector", "meta_synthesis", "state_creative"]),
("ТОПОР", 152, 153, ["will_vector", "meta_synthesis", "state_destructive"]),
("КЛЮЧ", 154, 155, ["meta_synthesis", "state_creative", "will_vector"]),
]
for sym_name, start_p, end_p, pref_fields in monographs:
chapter_text = []
chapter_imgs = []
for p in range(start_p - 1, min(end_p, len(doc))):
chapter_text.append(doc[p].get_text())
imgs = extract_images_from_page(doc, p, "karelin", sym_name, f"Монография: {sym_name}")
chapter_imgs.extend(imgs)
full_chapter_txt = clean_spaces(" ".join(chapter_text))
if len(full_chapter_txt) > 100:
route_symbol(
term=sym_name,
author="В.В. Карелин",
work="Лабиринт. Дракон. Единорог… О чем говорят символы",
text=full_chapter_txt[:1200],
preferred_fields=pref_fields,
illustrations=chapter_imgs
)
stats["karelin_entries"] += 1
print(f" Обработано монографий Карелина: {stats['karelin_entries']}.")
# =========================================================================
# 3. Н.Д. КОРТУНОВА — «ЗНАКИ, СИМВОЛЫ И АЛЛЕГОРИИ В ИСКУССТВЕ»
# =========================================================================
kortunova_path = NEWSOURCES_DIR / "Кортунова Н. Д. - Знаки, символы и аллегории в искусстве (Искусство понимать) - 2023.pdf"
if kortunova_path.exists():
print("\n📖 [3/8] Обработка Н.Д. Кортунова — «Знаки, символы и аллегории в искусстве»...")
doc = fitz.open(kortunova_path)
for pno in range(15, min(185, len(doc))):
raw_txt = doc[pno].get_text()
if not raw_txt or len(raw_txt) < 80:
continue
lines = [l.strip() for l in raw_txt.split('\n') if l.strip()]
candidate_term = None
for l in lines[:4]:
if len(l) < 25 and re.match(r'^[А-ЯЁ][а-яё\- ]+$', l):
if l not in ["Глава I", "Глава II", "Глава III", "Глава IV", "Флора", "Фауна", "Мир предметов", "Аллегории", "Знаки, символы и аллегории в искусстве"]:
candidate_term = l
break
if candidate_term:
clean_body = clean_spaces(raw_txt)
page_imgs = extract_images_from_page(doc, pno, "kortunova", candidate_term, f"Шедевр искусства: {candidate_term}")
route_symbol(
term=candidate_term,
author="Н.Д. Кортунова",
work="Знаки, символы и аллегории в искусстве",
text=clean_body[:800],
preferred_fields=["culture_antiquity_light", "culture_hermetic_light", "semantic_core"],
illustrations=page_imgs
)
stats["kortunova_entries"] += 1
print(f" Обработано разделов Кортуновой: {stats['kortunova_entries']}.")
# =========================================================================
# 4. К.Г. ЮНГ — ТРИ ТОМА АРХЕТИПИЧЕСКОЙ СИМВОЛИКИ
# =========================================================================
jung_books = [
("Символы трансформации", NEWSOURCES_DIR / "Карл Густав Юнг - Символы трансформации (Психологические технологии) - 2020.pdf"),
("Символы сновидений", NEWSOURCES_DIR / "Карл Густав Юнг - Символы сновидений в процессе индивидуации - 2021.pdf"),
("Человек и его символы", NEWSOURCES_DIR / "Юнг К. Г. - Человек и его символы - 2020.pdf"),
]
jung_archetypes = [
("ГЕРОЙ", ["state_creative", "will_vector", "meta_synthesis"]),
("МАТЬ", ["cosmogonic_role", "meta_synthesis", "state_creative"]),
("ЗМЕЙ", ["chthonic_aspect", "state_destructive", "meta_synthesis"]),
("ДРАКОН", ["chthonic_aspect", "state_destructive", "meta_synthesis"]),
("ДРЕВО", ["cosmogonic_role", "meta_synthesis", "state_creative"]),
("ВОДА", ["elemental_affinity", "state_creative", "meta_synthesis"]),
("ОГОНЬ", ["elemental_affinity", "state_creative", "meta_synthesis"]),
("СОЛНЦЕ", ["solar_lunar_axis", "state_creative", "meta_synthesis"]),
("ЖЕРТВА", ["state_destructive", "will_vector", "meta_synthesis"]),
("ТЕНЬ", ["state_destructive", "shadow_warning", "meta_synthesis"]),
("ЗОЛОТО", ["culture_hermetic_light", "meta_synthesis", "state_creative"]),
("КАМЕНЬ", ["culture_hermetic_light", "meta_synthesis", "state_creative"]),
("МАНДАЛА", ["geometric_archetype", "meta_synthesis", "state_creative"]),
("САМОСТЬ", ["meta_synthesis", "state_creative", "semantic_core"]),
("УРОБОРОС", ["geometric_archetype", "meta_synthesis", "chthonic_aspect"]),
("ЛАБИРИНТ", ["geometric_archetype", "meta_synthesis", "state_creative"]),
("КОНЬ", ["archetype_category", "state_creative", "meta_synthesis"]),
("РЫБА", ["archetype_category", "state_creative", "meta_synthesis"]),
("ПТИЦА", ["archetype_category", "state_creative", "meta_synthesis"]),
]
print("\n📖 [4/8] Обработка наследия К.Г. Юнга...")
for j_title, j_file in jung_books:
if not j_file.exists():
continue
doc = fitz.open(j_file)
for pno in range(min(500, len(doc))):
page_text = doc[pno].get_text()
if not page_text or len(page_text) < 100:
continue
for arch, p_fields in jung_archetypes:
pattern = rf'\b{arch.capitalize()}[а-яё]*\b[^\.\n]*?(?:архетип|символ|бессознательн|либидо|трансформац|сновиден)[^\.\n]*?\.'
m = re.search(pattern, page_text, re.IGNORECASE)
if m:
snippet = page_text[max(0, m.start() - 100):min(len(page_text), m.end() + 350)]
clean_snip = clean_spaces(snippet)
if len(clean_snip) > 70:
route_symbol(
term=arch,
author="К.Г. Юнг",
work=j_title,
text=clean_snip,
preferred_fields=p_fields,
illustrations=[]
)
stats["jung_entries"] += 1
break
print(f" Извлечено цитат Юнга: {stats['jung_entries']}.")
# =========================================================================
# 5. В.В. БАЙДИН — «АРХЕТИПЫ И СИМВОЛЫ РУССКОЙ КУЛЬТУРЫ» (FB2)
# =========================================================================
baydin_path = NEWSOURCES_DIR / "Байдин В.В. - Архетипы и символы русской культуры. От архаики до современности - 2021" / "Байдин В.В. - Архетипы и символы русской культуры - 2021.fb2"
if baydin_path.exists():
print("\n📖 [5/8] Обработка В.В. Байдин — «Архетипы и символы русской культуры»...")
try:
with open(baydin_path, "r", encoding="utf-8", errors="ignore") as f:
fb2_content = f.read()
slavic_archetypes = [
("СОЛНЦЕ", ["culture_slavic_light", "script_slavic"]),
("ОГОНЬ", ["culture_slavic_light", "script_slavic"]),
("ЗЕМЛЯ", ["culture_slavic_light", "script_slavic"]),
("ВОДА", ["culture_slavic_light", "script_slavic"]),
("ДЕРЕВО", ["culture_slavic_light", "script_slavic"]),
("ДУБ", ["culture_slavic_light", "script_slavic"]),
("КОНЬ", ["culture_slavic_light", "script_slavic"]),
("ПТИЦА", ["culture_slavic_light", "script_slavic"]),
("ХРАМ", ["culture_slavic_light", "script_slavic"]),
("КРЕСТ", ["culture_slavic_light", "script_slavic"]),
("КОЛОВРАТ", ["culture_slavic_light", "script_slavic"]),
("ЗМЕЙ", ["culture_slavic_shadow", "script_slavic"]),
]
for s_term, s_fields in slavic_archetypes:
pattern = rf'[^<]*?\b{s_term.capitalize()}[а-яё]*\b[^<]*?(?:славян|святилищ|архаик|курган|культ|древнерус)[^<]*?
'
matches = re.findall(pattern, fb2_content, re.IGNORECASE)
for raw_p in matches[:3]:
clean_p = clean_spaces(re.sub(r'<[^>]+>', '', raw_p))
if len(clean_p) > 50:
route_symbol(
term=s_term,
author="В.В. Байдин",
work="Архетипы и символы русской культуры",
text=clean_p,
preferred_fields=s_fields,
illustrations=[]
)
stats["baydin_entries"] += 1
print(f" Извлечено славянских архетипов Байдина: {stats['baydin_entries']}.")
except Exception as e:
print(f" Ошибка чтения Байдина: {e}")
# =========================================================================
# 6. Б. ГОФФ — «SYMBOLS OF PREHISTORIC MESOPOTAMIA» (498 стр.)
# =========================================================================
goff_path = NEWSOURCES_DIR / "_Goff B. Symbols of prehistoric Mesopotamia (1963).pdf"
if goff_path.exists():
print("\n📖 [6/8] Обработка Б. Гофф — «Symbols of Prehistoric Mesopotamia»...")
doc = fitz.open(goff_path)
meso_motifs = [
("РОЗЕТКА", "rosette", ["culture_sumer_light", "script_sumer_akkad"]),
("БЫК", "bull", ["culture_sumer_light", "script_sumer_akkad"]),
("ЗМЕЯ", "serpent", ["culture_sumer_light", "script_sumer_akkad"]),
("ДРЕВО", "tree", ["culture_sumer_light", "script_sumer_akkad"]),
("СОСУД", "vase", ["culture_sumer_light", "script_sumer_akkad"]),
("ЛЕВ", "lion", ["culture_sumer_light", "script_sumer_akkad"]),
("РОГА", "horns", ["culture_sumer_light", "script_sumer_akkad"]),
]
for pno in range(30, min(300, len(doc))):
raw_text = doc[pno].get_text()
if not raw_text or len(raw_text) < 100:
continue
for ru_term, en_term, s_fields in meso_motifs:
if en_term in raw_text.lower():
sentences = re.findall(rf'[^.!?]*\b{en_term}\b[^.!?]*[.!?]', raw_text, re.IGNORECASE)
if sentences:
eng_text = clean_spaces(" ".join(sentences[:2]))
page_imgs = extract_images_from_page(doc, pno, "goff", ru_term, f"Prehistoric Mesopotamian: {en_term.capitalize()}")
route_symbol(
term=ru_term,
author="Б. Гофф (Beatrice Goff)",
work="Symbols of Prehistoric Mesopotamia",
text=eng_text,
preferred_fields=s_fields,
illustrations=page_imgs
)
stats["goff_entries"] += 1
break
print(f" Извлечено месопотамских мотивов Гофф: {stats['goff_entries']}.")
# =========================================================================
# 7. ТИМОФЕЙ ИЗ ГАЗЫ — «ТИГРИЦА И ГРИФОН» (БЕСТИАРИЙ)
# =========================================================================
timofey_path = NEWSOURCES_DIR / "Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002" / "Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002.pdf"
if timofey_path.exists():
print("\n📖 [7/8] Обработка Тимофей из Газы — «Тигрица и грифон»...")
doc = fitz.open(timofey_path)
bestiary_animals = [
("ЛЕВ", ["culture_antiquity_light", "archetype_category"]),
("ОРЕЛ", ["culture_antiquity_light", "archetype_category"]),
("ОЛЕНЬ", ["culture_antiquity_light", "archetype_category"]),
("ДЕЛЬФИН", ["culture_antiquity_light", "archetype_category"]),
("СЛОН", ["culture_antiquity_light", "archetype_category"]),
("СКОРПИОН", ["chthonic_aspect", "culture_antiquity_shadow"]),
("ПАНТЕРА", ["culture_antiquity_light", "archetype_category"]),
("ГРИФОН", ["archetype_category", "meta_synthesis"]),
("ВАСИЛИСК", ["chthonic_aspect", "state_destructive"]),
("ФЕНИКС", ["state_creative", "meta_synthesis"]),
("ВОРОН", ["culture_antiquity_light", "archetype_category"]),
("СОВА", ["culture_antiquity_light", "archetype_category"]),
("ПАУК", ["chthonic_aspect", "culture_antiquity_shadow"]),
("ПЧЕЛА", ["state_creative", "culture_antiquity_light"]),
]
for pno in range(20, min(350, len(doc))):
raw_text = doc[pno].get_text()
if not raw_text or len(raw_text) < 100:
continue
for animal, a_fields in bestiary_animals:
m = re.search(rf'\b{animal.capitalize()}[а-яё]*\b[^\.\n]*?(?:сакральн|символ|природ|звер|почитан|божеств)[^\.\n]*?\.', raw_text, re.IGNORECASE)
if m:
snippet = clean_spaces(raw_text[max(0, m.start() - 80):min(len(raw_text), m.end() + 300)])
page_imgs = extract_images_from_page(doc, pno, "timofey", animal, f"Бестиарий: {animal}")
route_symbol(
term=animal,
author="Тимофей из Газы",
work="Тигрица и грифон. Сакральные символы животного мира",
text=snippet,
preferred_fields=a_fields,
illustrations=page_imgs
)
stats["timofey_entries"] += 1
break
print(f" Извлечено статей бестиария Тимофея: {stats['timofey_entries']}.")
# =========================================================================
# 8. Р. ШТАЙНЕР — «ОККУЛЬТНЫЕ ЗНАКИ И СИМВОЛЫ» (FB2)
# =========================================================================
steiner_path = NEWSOURCES_DIR / "Rudolf_Shtajner__Okkultnye_znaki_i_simvoly_Lekcii_14.fb2"
if steiner_path.exists():
print("\n📖 [8/8] Обработка Р. Штайнер — «Оккультные знаки и символы»...")
try:
with open(steiner_path, "r", encoding="utf-8", errors="ignore") as f:
fb2_steiner = f.read()
steiner_symbols = [
("ПЕНТАГРАММА", ["geometric_archetype", "meta_synthesis"]),
("ГЕКСАГРАММА", ["geometric_archetype", "meta_synthesis"]),
("СВАСТИКА", ["geometric_archetype", "meta_synthesis"]),
("КРЕСТ", ["meta_synthesis", "culture_hermetic_light"]),
("РОЗА", ["meta_synthesis", "culture_hermetic_light"]),
]
for st_term, st_fields in steiner_symbols:
pattern = rf'[^<]*?\b{st_term.capitalize()}[а-яё]*\b[^<]*?(?:оккультн|духовн|знак|символ|микрокосм|макрокосм)[^<]*?
'
matches = re.findall(pattern, fb2_steiner, re.IGNORECASE)
for raw_p in matches[:3]:
clean_p = clean_spaces(re.sub(r'<[^>]+>', '', raw_p))
if len(clean_p) > 60:
route_symbol(
term=st_term,
author="Р. Штайнер",
work="Оккультные знаки и символы",
text=clean_p,
preferred_fields=st_fields,
illustrations=[]
)
stats["steiner_entries"] += 1
print(f" Извлечено символов Штайнера: {stats['steiner_entries']}.")
except Exception as e:
print(f" Ошибка чтения Штайнера: {e}")
# =========================================================================
# СОХРАНЕНИЕ РЕЗУЛЬТАТОВ
# =========================================================================
print("\n💾 Сохранение баз данных...")
with open(STAGING_DB_PATH, "w", encoding="utf-8") as f:
json.dump(staging_db, f, ensure_ascii=False, indent=2)
print(f"✅ Staging база сохранена: {STAGING_DB_PATH}")
MAIN_DB_PATH = PROJECT_ROOT / "assets" / "data" / "oracle_passports.json"
with open(MAIN_DB_PATH, "w", encoding="utf-8") as f:
json.dump(staging_db, f, ensure_ascii=False, indent=2)
print(f"✅ Основная база синхронизирована: {MAIN_DB_PATH}")
with open(EXTERNAL_DB_PATH, "w", encoding="utf-8") as f:
json.dump(external_db, f, ensure_ascii=False, indent=2)
print(f"✅ База внешних символов сохранена: {EXTERNAL_DB_PATH} ({len(external_db)} терминов)")
with open(ENRICHER_EXTERNAL_PATH, "w", encoding="utf-8") as f:
json.dump(external_db, f, ensure_ascii=False, indent=2)
print(f"✅ Копия в oracle_enricher сохранена: {ENRICHER_EXTERNAL_PATH}")
with open(CATALOG_PATH, "w", encoding="utf-8") as f:
json.dump(catalog, f, ensure_ascii=False, indent=2)
print(f"✅ Каталог иллюстраций обновлен: {CATALOG_PATH} ({len(catalog)} изображений)")
print("\n" + "=" * 70)
print("📊 ИТОГОВЫЙ ОТЧЕТ ИНГЕСТИИ:")
print(f" • Обогащено ячеек Оракула 999: +{stats['core_enriched']}")
print(f" • Добавлено внешних символов: +{stats['external_added']}")
print(f" • Всего внешних символов в базе: {len(external_db)}")
print(f" • Новых иллюстраций извлечено: +{stats['illustrations_added']}")
print(f" • Всего иллюстраций в каталоге: {len(catalog)}")
print("=" * 70)
if __name__ == "__main__":
main()