import re import json from pathlib import Path import fitz BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999') ILLUST_DIR = BASE_DIR / 'assets/illustrations' CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json' EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json' PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json' STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json' PDF_PATH = Path('/home/alexives/Desktop/newsources/Уильямс Ч. - Китайская культура. Мифы, герои, символы - 2011.pdf') ru_to_en = { 'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo', 'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm', 'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u', 'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'shch', 'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya' } def slugify(text: str) -> str: slug = [] for c in text.lower(): if c in ru_to_en: slug.append(ru_to_en[c]) elif c.isalnum(): slug.append(c) elif c in [' ', '-', '_']: slug.append('_') res = re.sub(r'_+', '_', ''.join(slug)).strip('_') return res[:30] if res else 'symbol' def run(): print("=== Master Williams Illustration Extractor & Linker ===") if not PDF_PATH.exists(): print(f"Error: PDF not found at {PDF_PATH}") return doc = fitz.open(str(PDF_PATH)) print(f"Opened Williams PDF: {len(doc)} pages") # 1. Parse all articles with page ranges # Collect all headers and pages article_spans = [] # (title, start_p, end_p) all_headers = [] for p in range(12, 470): txt = doc[p].get_text() lines = [l.strip() for l in txt.splitlines() if l.strip()] for l in lines: if l.isupper() and 2 <= len(l) <= 35 and re.match(r'^[А-ЯЁ\s\-]+$', l): if not any(skip in l for skip in ['КИТАЙСКАЯ КУЛЬТУРА', 'СОДЕРЖАНИЕ', 'ВВЕДЕНИЕ', 'УКАЗАТЕЛЬ', 'ЧАСТЬ', 'ПРИЛОЖЕНИЕ']): all_headers.append((p, l)) # De-duplicate consecutive headers on same page filtered_headers = [] for p, h in all_headers: if not filtered_headers or filtered_headers[-1][1] != h: filtered_headers.append((p, h)) print(f"Found {len(filtered_headers)} article headers in Williams") # Map each page to the active article header page_to_article = {} for i, (p, h) in enumerate(filtered_headers): next_p = filtered_headers[i+1][0] if i+1 < len(filtered_headers) else 470 for pg in range(p, next_p + 1): if pg not in page_to_article: page_to_article[pg] = h # 2. Load existing catalog and external symbols with open(CATALOG_PATH, 'r', encoding='utf-8') as f: catalog = json.load(f) existing_catalog_files = set(x.get('file') for x in catalog) with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f: ext_symbols = json.load(f) with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f: passports = json.load(f) # Build core word lookup: normalized_word -> passport_id core_lookup = {} for pid, pdata in passports.items(): w = pdata.get('word', '').strip().lower() if w: core_lookup[w] = pid new_extracted_count = 0 new_catalog_items = [] symbol_to_illustrations = {} # normalized_symbol -> list of ill dicts # 3. Extract images across all pages for p in range(12, 470): active_header = page_to_article.get(p) if not active_header: continue slug = slugify(active_header) norm_title = active_header.lower() imgs = doc[p].get_images() if not imgs: continue for idx, img_info in enumerate(imgs): xref = img_info[0] try: base_img = doc.extract_image(xref) w, h = base_img["width"], base_img["height"] ext = base_img["ext"] # Extract both large illustrations (woodcuts) and medium glyphs/seals is_large_art = (w * h >= 20000) or (max(w, h) >= 160 and min(w, h) >= 90) is_sacred_glyph = (w >= 70 and h >= 70 and w * h >= 5000) if not (is_large_art or is_sacred_glyph): continue fname = f"wil_{p:03d}_{slug}_{idx}.{ext}" rel_path = f"assets/illustrations/{fname}" abs_path = ILLUST_DIR / fname if not abs_path.exists(): with open(abs_path, 'wb') as f_out: f_out.write(base_img['image']) new_extracted_count += 1 ill_id = f"wil_{p}_{idx}" caption_type = "Гравюра и образ" if is_large_art else "Сакральное начертание / Глиф" caption = f"Ч. Уильямс · {active_header.title()} ({caption_type})" is_core = norm_title in core_lookup core_id = core_lookup.get(norm_title) ill_entry = { "id": ill_id, "file": rel_path, "symbol": active_header, "normalized": norm_title, "caption": caption, "source": "Ч. Уильямс · «Китайская культура: символы и мифы» (2011)", "page": p + 1, "width": w, "height": h, "is_core_1000": is_core, "core_id": str(core_id) if core_id else None } if rel_path not in existing_catalog_files: catalog.append(ill_entry) existing_catalog_files.add(rel_path) new_catalog_items.append(ill_entry) if norm_title not in symbol_to_illustrations: symbol_to_illustrations[norm_title] = [] # Avoid duplicate files for same symbol if not any(x['file'] == rel_path for x in symbol_to_illustrations[norm_title]): symbol_to_illustrations[norm_title].append(ill_entry) except Exception as e: continue print(f"Extracted {new_extracted_count} new images to assets/illustrations/") print(f"Added {len(new_catalog_items)} new items to catalog (total in catalog: {len(catalog)})") # 4. Attach to external symbols updated_ext_count = 0 for norm_title, ills in symbol_to_illustrations.items(): # Match in external symbols (exact or subkey) target_keys = [] if norm_title in ext_symbols: target_keys.append(norm_title) else: for k in ext_symbols: if k == norm_title or norm_title.startswith(k + ' ') or k.startswith(norm_title + ' '): target_keys.append(k) for tk in target_keys: existing_ills = ext_symbols[tk].get('illustrations', []) existing_files = set(x.get('file') for x in existing_ills) added = 0 for ill in ills: if ill['file'] not in existing_files: existing_ills.append(ill) existing_files.add(ill['file']) added += 1 if added > 0: ext_symbols[tk]['illustrations'] = existing_ills updated_ext_count += 1 print(f"Updated {updated_ext_count} external symbols with Williams illustrations") # 5. Attach to Core 1000 passports updated_passports_count = 0 for norm_title, ills in symbol_to_illustrations.items(): if norm_title in core_lookup: pid = core_lookup[norm_title] p_data = passports[pid] existing_ills = p_data.get('illustrations', []) existing_files = set(x.get('file') for x in existing_ills) added = 0 for ill in ills: if ill['file'] not in existing_files: existing_ills.append(ill) existing_files.add(ill['file']) added += 1 if added > 0: p_data['illustrations'] = existing_ills updated_passports_count += 1 print(f"Updated {updated_passports_count} Core 1000 passports with Williams illustrations") # Save everything back to disk with open(CATALOG_PATH, 'w', encoding='utf-8') as f: json.dump(catalog, f, ensure_ascii=False, indent=2) with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f: json.dump(ext_symbols, f, ensure_ascii=False, indent=2) with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f: json.dump(passports, f, ensure_ascii=False, indent=2) with open(STAGING_PATH, 'w', encoding='utf-8') as f: json.dump(passports, f, ensure_ascii=False, indent=2) print("=== Successfully finished Williams Master Enrichment! ===") if __name__ == '__main__': run()