448 lines
18 KiB
Python
448 lines
18 KiB
Python
import re
|
||
import json
|
||
import time
|
||
from pathlib import Path
|
||
import fitz
|
||
|
||
BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
|
||
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
|
||
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
|
||
STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
|
||
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
|
||
ILLUST_DIR = BASE_DIR / 'assets/illustrations'
|
||
|
||
NEWSOURCES_DIR = Path('/home/alexives/Desktop/newsources')
|
||
COOPER_PDF = NEWSOURCES_DIR / 'Kuper_Dzh_Entsiklopedia_simvolov.pdf'
|
||
KORTUNOVA_PDF = NEWSOURCES_DIR / 'Кортунова Н. Д. - Знаки, символы и аллегории в искусстве (Искусство понимать) - 2023.pdf'
|
||
TIMOFEY_PDF = NEWSOURCES_DIR / 'Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002' / 'Тимофей из Газы - Тигрица и грифон. Сакральные символы животного мира-2002.pdf'
|
||
|
||
t_start = time.time()
|
||
print("=" * 80)
|
||
print("🚀 FAST COMPREHENSIVE REBUILD: COOPER, KORTUNOVA, TIMOTHY OF GAZA")
|
||
print("=" * 80)
|
||
|
||
def dehyphen(text: str) -> str:
|
||
return re.sub(r'([А-Яа-яA-Za-zЁё]+)[-\xad\u2010\u2011\u2012\u2013\u2014]\s*\n\s*([А-Яа-яA-Za-zЁё]+)', r'\1\2', text)
|
||
|
||
def clean_spaces(text: str) -> str:
|
||
return re.sub(r'\s+', ' ', text).strip()
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 1. LOAD DATABASES
|
||
# -------------------------------------------------------------------------
|
||
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
|
||
ext = json.load(f)
|
||
with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
|
||
passports = json.load(f)
|
||
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
|
||
catalog = json.load(f)
|
||
|
||
core_lookup = {}
|
||
for pid, p in passports.items():
|
||
w = p.get('word', '').strip().lower()
|
||
if w:
|
||
core_lookup[w] = pid
|
||
|
||
print(f"Loaded existing databases in {time.time()-t_start:.2f}s")
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 2. INGEST COOPER (1,100+ ARTICLES, 165 ENGRAVINGS)
|
||
# -------------------------------------------------------------------------
|
||
t_step = time.time()
|
||
doc_c = fitz.open(str(COOPER_PDF))
|
||
|
||
# Extract engravings with Рис. <N>
|
||
cooper_illustrations = {}
|
||
for pno in range(6, len(doc_c)-1):
|
||
txt = doc_c[pno].get_text()
|
||
fig_matches = list(re.finditer(r'Рис\.\s*(\d+)[\.:]?\s*\n?([^\n]+(?:\n[^\n]+)?)', txt))
|
||
imgs = doc_c[pno].get_images()
|
||
if fig_matches and imgs:
|
||
extracted = []
|
||
for idx, fm in enumerate(fig_matches):
|
||
if idx < len(imgs):
|
||
f_num = fm.group(1)
|
||
raw_cap = fm.group(2).replace('\n', ' ').strip()
|
||
cap_clean = clean_spaces(raw_cap)[:140]
|
||
xref = imgs[idx][0]
|
||
try:
|
||
base = doc_c.extract_image(xref)
|
||
w, h = base["width"], base["height"]
|
||
ext_fmt = base["ext"]
|
||
if w >= 140 and h >= 140:
|
||
fname = f"cooper_fig_{int(f_num):03d}_p{pno+1}.{ext_fmt}"
|
||
out_path = ILLUST_DIR / fname
|
||
if not out_path.exists():
|
||
with open(out_path, "wb") as f_out:
|
||
f_out.write(base["image"])
|
||
extracted.append({
|
||
"id": f"cooper_fig_{f_num}",
|
||
"file": f"assets/illustrations/{fname}",
|
||
"caption": f"Рис. {f_num}. {cap_clean}",
|
||
"page": pno + 1,
|
||
"width": w,
|
||
"height": h,
|
||
"source": "Дж. Купер · «Энциклопедия символов» (1995)"
|
||
})
|
||
except Exception:
|
||
pass
|
||
if extracted:
|
||
cooper_illustrations[pno] = extracted
|
||
|
||
print(f"Cooper illustrations extracted: {sum(len(v) for v in cooper_illustrations.values())} in {time.time()-t_step:.2f}s")
|
||
|
||
# Extract articles block by block
|
||
cooper_articles = {}
|
||
stop_terms = {
|
||
'энциклопедия символов', 'иллюстрации', 'содержание', 'указатель',
|
||
'латинские термины', 'библиография', 'введение', 'предисловие', 'рис'
|
||
}
|
||
|
||
ocr_fixes = {
|
||
'абоикос': 'абрикос',
|
||
'акадия': 'акация',
|
||
'айвя': 'айва',
|
||
'амис': 'апис',
|
||
}
|
||
|
||
cur_term = None
|
||
cur_lines = []
|
||
cur_pno = 0
|
||
|
||
for pno in range(6, len(doc_c)-1):
|
||
blocks = doc_c[pno].get_text('blocks')
|
||
for b in blocks:
|
||
txt = dehyphen(b[4]).strip()
|
||
m = re.match(r'^([А-ЯЁ][А-Яа-яЁё\s\-]{1,25}?)(?:\.|\s+—|\s+-)\s+(.*)', txt, re.DOTALL)
|
||
if m:
|
||
cand = m.group(1).strip()
|
||
rest = m.group(2).strip()
|
||
norm = cand.lower()
|
||
norm = ocr_fixes.get(norm, norm)
|
||
if (norm not in stop_terms and
|
||
not any(c.isdigit() for c in norm) and
|
||
1 <= len(norm.split()) <= 3 and
|
||
not norm.startswith(('см.', 'рис.', 'в ', 'у ', 'на ', 'по ', 'от ', 'для ', 'как ', 'но ', 'он ', 'она ', 'они '))):
|
||
if cur_term and cur_lines:
|
||
full_text = clean_spaces(' '.join(cur_lines))
|
||
if len(full_text) >= 30:
|
||
norm_k = ocr_fixes.get(cur_term.lower(), cur_term.lower())
|
||
if norm_k not in cooper_articles or len(full_text) > len(cooper_articles[norm_k]['text']):
|
||
cooper_articles[norm_k] = {
|
||
'term': cur_term,
|
||
'text': full_text,
|
||
'page': cur_pno,
|
||
'illustrations': cooper_illustrations.get(cur_pno, [])
|
||
}
|
||
cur_term = cand.capitalize()
|
||
cur_pno = pno
|
||
cur_lines = [rest] if rest else []
|
||
continue
|
||
if cur_term:
|
||
# Body block
|
||
clean_b = ' '.join(l for l in txt.splitlines() if not l.strip().startswith('Рис.'))
|
||
if clean_b:
|
||
cur_lines.append(clean_b)
|
||
|
||
if cur_term and cur_lines:
|
||
full_text = clean_spaces(' '.join(cur_lines))
|
||
if len(full_text) >= 30:
|
||
norm_k = ocr_fixes.get(cur_term.lower(), cur_term.lower())
|
||
cooper_articles[norm_k] = {
|
||
'term': cur_term,
|
||
'text': full_text,
|
||
'page': cur_pno,
|
||
'illustrations': cooper_illustrations.get(cur_pno, [])
|
||
}
|
||
|
||
print(f"Cooper articles parsed: {len(cooper_articles)} in {time.time()-t_step:.2f}s")
|
||
|
||
# Integrate Cooper
|
||
for norm_k, item in cooper_articles.items():
|
||
if norm_k not in ext:
|
||
ext[norm_k] = {
|
||
'term': item['term'].upper(),
|
||
'normalized': norm_k,
|
||
'category_guess': 'Академический универсальный символ',
|
||
'sources': [],
|
||
'illustrations': [],
|
||
'created_at': '2026-10-08T20:00:00',
|
||
'updated_at': '2026-10-08T20:00:00'
|
||
}
|
||
|
||
ext[norm_k]['sources'].append({
|
||
'author': 'Дж. Купер',
|
||
'work': '«Энциклопедия символов» (1995)',
|
||
'excerpt': item['text'][:350] + '…' if len(item['text']) > 350 else item['text'],
|
||
'full_text': item['text']
|
||
})
|
||
|
||
for ill in item['illustrations']:
|
||
if not any(x.get('file') == ill['file'] for x in ext[norm_k].get('illustrations', [])):
|
||
ext[norm_k]['illustrations'].append(ill)
|
||
catalog.append({
|
||
'id': ill['id'],
|
||
'file': ill['file'],
|
||
'symbol': item['term'].upper(),
|
||
'normalized': norm_k,
|
||
'caption': ill['caption'],
|
||
'source': ill['source'],
|
||
'page': ill['page'],
|
||
'width': ill['width'],
|
||
'height': ill['height'],
|
||
'is_core_1000': norm_k in core_lookup,
|
||
'core_id': core_lookup.get(norm_k)
|
||
})
|
||
|
||
if norm_k in core_lookup:
|
||
pid = core_lookup[norm_k]
|
||
p_item = passports[pid]
|
||
for ill in item['illustrations']:
|
||
if not any(x.get('file') == ill['file'] for x in p_item.get('illustrations', [])):
|
||
p_item['illustrations'].append(ill)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 3. INGEST KORTUNOVA (39 CANONICAL CHAPTERS & MASTERPIECES)
|
||
# -------------------------------------------------------------------------
|
||
t_step = time.time()
|
||
doc_k = fitz.open(str(KORTUNOVA_PDF))
|
||
|
||
kortunova_sections = [
|
||
("Гранат", 16, 17),
|
||
("Виноград", 18, 19),
|
||
("Плющ", 20, 23),
|
||
("Лимон", 24, 27),
|
||
("Земляника", 28, 36),
|
||
("Лавр", 37, 47),
|
||
("Пальма", 48, 51),
|
||
("Олива", 52, 54),
|
||
("Кипарис", 55, 56),
|
||
("Мирт", 57, 65),
|
||
("Голубь", 66, 68),
|
||
("Павлин", 69, 72),
|
||
("Сова", 73, 77),
|
||
("Рыба", 78, 87),
|
||
("Змея", 88, 90),
|
||
("Олень", 91, 96),
|
||
("Лев", 97, 99),
|
||
("Обезьяна", 100, 105),
|
||
("Ягненок", 106, 107),
|
||
("Орел", 108, 108),
|
||
("Петух", 109, 111),
|
||
("Кролик", 112, 117),
|
||
("Лошадь", 118, 123),
|
||
("Жемчуг", 124, 128),
|
||
("Зеркало", 129, 130),
|
||
("Раковина", 131, 135),
|
||
("Лестница", 136, 141),
|
||
("Корона", 142, 142),
|
||
("Коралл", 143, 146),
|
||
("Мантия", 147, 153),
|
||
("Времена года", 154, 157),
|
||
("Весна", 158, 158),
|
||
("Лето", 159, 159),
|
||
("Осень", 160, 160),
|
||
("Зима", 161, 163),
|
||
("Четыре элемента", 164, 167),
|
||
("Пять чувств", 168, 173),
|
||
("Четыре части света", 174, 174),
|
||
("Пороки и добродетели", 175, 185),
|
||
]
|
||
|
||
for sym_title, start_p, end_p in kortunova_sections:
|
||
norm_k = sym_title.lower()
|
||
col_text = []
|
||
col_ills = []
|
||
|
||
for pno in range(start_p - 1, min(end_p, len(doc_k))):
|
||
col_text.append(dehyphen(doc_k[pno].get_text()))
|
||
for idx, img in enumerate(doc_k[pno].get_images()):
|
||
try:
|
||
base = doc_k.extract_image(img[0])
|
||
w, h = base["width"], base["height"]
|
||
ext_fmt = base["ext"]
|
||
if w >= 200 and h >= 200:
|
||
fname = f"kortunova_{norm_k}_{pno+1}_{idx}_{w}x{h}.{ext_fmt}"
|
||
out_path = ILLUST_DIR / fname
|
||
if not out_path.exists():
|
||
with open(out_path, "wb") as f_out:
|
||
f_out.write(base["image"])
|
||
col_ills.append({
|
||
"id": f"kortunova_{pno}_{idx}",
|
||
"file": f"assets/illustrations/{fname}",
|
||
"caption": f"Шедевр искусства: {sym_title} (стр. {pno+1})",
|
||
"page": pno + 1,
|
||
"width": w,
|
||
"height": h,
|
||
"source": "Н.Д. Кортунова · «Знаки, символы и аллегории в искусстве» (2023)"
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
full_body = clean_spaces(' '.join(col_text))
|
||
full_body = re.sub(r'Знаки, символы и аллегории в искусстве|Глава I|Глава II|Глава III|Глава IV|Флора|Фауна|Мир предметов|Аллегории|\b\d{1,3}\b', ' ', full_body)
|
||
full_body = clean_spaces(full_body)
|
||
|
||
if norm_k not in ext:
|
||
ext[norm_k] = {
|
||
'term': sym_title.upper(),
|
||
'normalized': norm_k,
|
||
'category_guess': 'Символ классического искусства',
|
||
'sources': [],
|
||
'illustrations': [],
|
||
'created_at': '2026-10-08T20:00:00',
|
||
'updated_at': '2026-10-08T20:00:00'
|
||
}
|
||
|
||
ext[norm_k]['sources'].append({
|
||
'author': 'Н.Д. Кортунова',
|
||
'work': '«Знаки, символы и аллегории в искусстве» (2023)',
|
||
'excerpt': full_body[:400] + '…' if len(full_body) > 400 else full_body,
|
||
'full_text': full_body
|
||
})
|
||
|
||
for ill in col_ills:
|
||
if not any(x.get('file') == ill['file'] for x in ext[norm_k].get('illustrations', [])):
|
||
ext[norm_k]['illustrations'].append(ill)
|
||
catalog.append({
|
||
'id': ill['id'],
|
||
'file': ill['file'],
|
||
'symbol': sym_title.upper(),
|
||
'normalized': norm_k,
|
||
'caption': ill['caption'],
|
||
'source': ill['source'],
|
||
'page': ill['page'],
|
||
'width': ill['width'],
|
||
'height': ill['height'],
|
||
'is_core_1000': norm_k in core_lookup,
|
||
'core_id': core_lookup.get(norm_k)
|
||
})
|
||
|
||
if norm_k in core_lookup:
|
||
pid = core_lookup[norm_k]
|
||
p_item = passports[pid]
|
||
for ill in col_ills:
|
||
if not any(x.get('file') == ill['file'] for x in p_item.get('illustrations', [])):
|
||
p_item['illustrations'].append(ill)
|
||
|
||
print(f"Kortunova parsed: 39 chapters and paintings in {time.time()-t_step:.2f}s")
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 4. INGEST TIMOTHY OF GAZA (57 ANIMAL BESTIARY CHAPTERS)
|
||
# -------------------------------------------------------------------------
|
||
t_step = time.time()
|
||
doc_t = fitz.open(str(TIMOFEY_PDF))
|
||
|
||
timofey_animals = [
|
||
("Гиена", 21, 30),
|
||
("Лисица", 31, 38),
|
||
("Еж", 39, 42),
|
||
("Волк", 43, 52),
|
||
("Дикобраз", 53, 54),
|
||
("Тигрица", 55, 62),
|
||
("Гриф", 63, 72),
|
||
("Барс", 73, 80),
|
||
("Леопард", 81, 84),
|
||
("Шакалы", 85, 86),
|
||
("Пантера", 87, 94),
|
||
("Дикая коза", 95, 102),
|
||
("Газель", 103, 104),
|
||
("Заяц", 105, 110),
|
||
("Черепаха", 111, 116),
|
||
("Бонасос", 117, 120),
|
||
("Онагр", 121, 124),
|
||
("Страус", 125, 128),
|
||
("Орикс", 129, 132),
|
||
("Жираф", 133, 138),
|
||
("Слон", 139, 148),
|
||
("Драконы", 149, 156),
|
||
("Собака", 157, 166),
|
||
("Лошадь", 167, 174),
|
||
("Бык", 175, 178),
|
||
("Буйвол", 179, 182),
|
||
("Мул", 183, 188),
|
||
("Осел", 189, 198),
|
||
("Верблюд", 199, 208),
|
||
("Кабан", 209, 218),
|
||
("Овца", 219, 222),
|
||
("Субос", 223, 224),
|
||
("Кошка", 225, 228),
|
||
("Куница", 229, 232),
|
||
("Мыши", 233, 238),
|
||
("Ласка", 239, 244),
|
||
("Землеройка", 245, 248),
|
||
("Крот", 249, 250),
|
||
("Сухопутный крокодил", 251, 256),
|
||
("Крокодил", 257, 266),
|
||
("Ихневмон", 267, 270),
|
||
("Гиппопотам", 271, 278),
|
||
("Носорог", 279, 286),
|
||
("Рысь", 287, 290),
|
||
("Хамелеон", 291, 294),
|
||
("Сцинк", 295, 300),
|
||
("Пятнистая гиена", 301, 304),
|
||
("Обезьяна", 305, 314),
|
||
("Сфингс", 315, 318),
|
||
("Катоблеп", 319, 322),
|
||
("Саламандра", 323, 326),
|
||
("Бобр", 327, 330),
|
||
("Лягушка", 331, 334),
|
||
("Тюлень", 335, 338),
|
||
("Раковина улитки", 339, 342),
|
||
("Зимородок", 343, 345)
|
||
]
|
||
|
||
for animal_title, start_p, end_p in timofey_animals:
|
||
norm_k = animal_title.lower()
|
||
col_text = []
|
||
|
||
for pno in range(start_p - 1, min(end_p, len(doc_t))):
|
||
col_text.append(dehyphen(doc_t[pno].get_text()))
|
||
|
||
full_body = clean_spaces(' '.join(col_text))
|
||
full_body = re.sub(r'Тимофей из Газы\. О животных|\b\d{1,3}\b', ' ', full_body)
|
||
full_body = clean_spaces(full_body)
|
||
|
||
if norm_k not in ext:
|
||
ext[norm_k] = {
|
||
'term': animal_title.upper(),
|
||
'normalized': norm_k,
|
||
'category_guess': 'Античный сакральный бестиарий',
|
||
'sources': [],
|
||
'illustrations': [],
|
||
'created_at': '2026-10-08T20:00:00',
|
||
'updated_at': '2026-10-08T20:00:00'
|
||
}
|
||
|
||
ext[norm_k]['sources'].append({
|
||
'author': 'Тимофей из Газы',
|
||
'work': '«О животных» (ок. 500 г. н. э. / 2002)',
|
||
'excerpt': full_body[:400] + '…' if len(full_body) > 400 else full_body,
|
||
'full_text': full_body
|
||
})
|
||
|
||
print(f"Timothy of Gaza parsed: 57 animal chapters in {time.time()-t_step:.2f}s")
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 5. SAVE EVERYTHING
|
||
# -------------------------------------------------------------------------
|
||
t_step = time.time()
|
||
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(ext, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(STAGING_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"Databases saved in {time.time()-t_step:.2f}s")
|
||
print("=" * 80)
|
||
print(f"🎉 MASTER REBUILD COMPLETE in {time.time()-t_start:.2f}s!")
|
||
print(f" • Чистых академических символов в базе: {len(ext)}")
|
||
print(f" • Иллюстраций в каталоге: {len(catalog)}")
|
||
print("=" * 80)
|