231 lines
9.1 KiB
Python
231 lines
9.1 KiB
Python
import re
|
||
import json
|
||
from pathlib import Path
|
||
import fitz
|
||
|
||
BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
|
||
ILLUST_DIR = BASE_DIR / 'assets/illustrations'
|
||
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
|
||
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
|
||
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
|
||
STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
|
||
PDF_PATH = Path('/home/alexives/Desktop/newsources/Уильямс Ч. - Китайская культура. Мифы, герои, символы - 2011.pdf')
|
||
|
||
ru_to_en = {
|
||
'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo',
|
||
'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm',
|
||
'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u',
|
||
'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'shch',
|
||
'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya'
|
||
}
|
||
|
||
def slugify(text: str) -> str:
|
||
slug = []
|
||
for c in text.lower():
|
||
if c in ru_to_en:
|
||
slug.append(ru_to_en[c])
|
||
elif c.isalnum():
|
||
slug.append(c)
|
||
elif c in [' ', '-', '_']:
|
||
slug.append('_')
|
||
res = re.sub(r'_+', '_', ''.join(slug)).strip('_')
|
||
return res[:30] if res else 'symbol'
|
||
|
||
def run():
|
||
print("=== Master Williams Illustration Extractor & Linker ===")
|
||
if not PDF_PATH.exists():
|
||
print(f"Error: PDF not found at {PDF_PATH}")
|
||
return
|
||
|
||
doc = fitz.open(str(PDF_PATH))
|
||
print(f"Opened Williams PDF: {len(doc)} pages")
|
||
|
||
# 1. Parse all articles with page ranges
|
||
# Collect all headers and pages
|
||
article_spans = [] # (title, start_p, end_p)
|
||
all_headers = []
|
||
|
||
for p in range(12, 470):
|
||
txt = doc[p].get_text()
|
||
lines = [l.strip() for l in txt.splitlines() if l.strip()]
|
||
for l in lines:
|
||
if l.isupper() and 2 <= len(l) <= 35 and re.match(r'^[А-ЯЁ\s\-]+$', l):
|
||
if not any(skip in l for skip in ['КИТАЙСКАЯ КУЛЬТУРА', 'СОДЕРЖАНИЕ', 'ВВЕДЕНИЕ', 'УКАЗАТЕЛЬ', 'ЧАСТЬ', 'ПРИЛОЖЕНИЕ']):
|
||
all_headers.append((p, l))
|
||
|
||
# De-duplicate consecutive headers on same page
|
||
filtered_headers = []
|
||
for p, h in all_headers:
|
||
if not filtered_headers or filtered_headers[-1][1] != h:
|
||
filtered_headers.append((p, h))
|
||
|
||
print(f"Found {len(filtered_headers)} article headers in Williams")
|
||
|
||
# Map each page to the active article header
|
||
page_to_article = {}
|
||
for i, (p, h) in enumerate(filtered_headers):
|
||
next_p = filtered_headers[i+1][0] if i+1 < len(filtered_headers) else 470
|
||
for pg in range(p, next_p + 1):
|
||
if pg not in page_to_article:
|
||
page_to_article[pg] = h
|
||
|
||
# 2. Load existing catalog and external symbols
|
||
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
|
||
catalog = json.load(f)
|
||
existing_catalog_files = set(x.get('file') for x in catalog)
|
||
|
||
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
|
||
ext_symbols = json.load(f)
|
||
|
||
with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
|
||
passports = json.load(f)
|
||
|
||
# Build core word lookup: normalized_word -> passport_id
|
||
core_lookup = {}
|
||
for pid, pdata in passports.items():
|
||
w = pdata.get('word', '').strip().lower()
|
||
if w:
|
||
core_lookup[w] = pid
|
||
|
||
new_extracted_count = 0
|
||
new_catalog_items = []
|
||
symbol_to_illustrations = {} # normalized_symbol -> list of ill dicts
|
||
|
||
# 3. Extract images across all pages
|
||
for p in range(12, 470):
|
||
active_header = page_to_article.get(p)
|
||
if not active_header:
|
||
continue
|
||
|
||
slug = slugify(active_header)
|
||
norm_title = active_header.lower()
|
||
imgs = doc[p].get_images()
|
||
if not imgs:
|
||
continue
|
||
|
||
for idx, img_info in enumerate(imgs):
|
||
xref = img_info[0]
|
||
try:
|
||
base_img = doc.extract_image(xref)
|
||
w, h = base_img["width"], base_img["height"]
|
||
ext = base_img["ext"]
|
||
|
||
# Extract both large illustrations (woodcuts) and medium glyphs/seals
|
||
is_large_art = (w * h >= 20000) or (max(w, h) >= 160 and min(w, h) >= 90)
|
||
is_sacred_glyph = (w >= 70 and h >= 70 and w * h >= 5000)
|
||
|
||
if not (is_large_art or is_sacred_glyph):
|
||
continue
|
||
|
||
fname = f"wil_{p:03d}_{slug}_{idx}.{ext}"
|
||
rel_path = f"assets/illustrations/{fname}"
|
||
abs_path = ILLUST_DIR / fname
|
||
|
||
if not abs_path.exists():
|
||
with open(abs_path, 'wb') as f_out:
|
||
f_out.write(base_img['image'])
|
||
new_extracted_count += 1
|
||
|
||
ill_id = f"wil_{p}_{idx}"
|
||
caption_type = "Гравюра и образ" if is_large_art else "Сакральное начертание / Глиф"
|
||
caption = f"Ч. Уильямс · {active_header.title()} ({caption_type})"
|
||
|
||
is_core = norm_title in core_lookup
|
||
core_id = core_lookup.get(norm_title)
|
||
|
||
ill_entry = {
|
||
"id": ill_id,
|
||
"file": rel_path,
|
||
"symbol": active_header,
|
||
"normalized": norm_title,
|
||
"caption": caption,
|
||
"source": "Ч. Уильямс · «Китайская культура: символы и мифы» (2011)",
|
||
"page": p + 1,
|
||
"width": w,
|
||
"height": h,
|
||
"is_core_1000": is_core,
|
||
"core_id": str(core_id) if core_id else None
|
||
}
|
||
|
||
if rel_path not in existing_catalog_files:
|
||
catalog.append(ill_entry)
|
||
existing_catalog_files.add(rel_path)
|
||
new_catalog_items.append(ill_entry)
|
||
|
||
if norm_title not in symbol_to_illustrations:
|
||
symbol_to_illustrations[norm_title] = []
|
||
# Avoid duplicate files for same symbol
|
||
if not any(x['file'] == rel_path for x in symbol_to_illustrations[norm_title]):
|
||
symbol_to_illustrations[norm_title].append(ill_entry)
|
||
|
||
except Exception as e:
|
||
continue
|
||
|
||
print(f"Extracted {new_extracted_count} new images to assets/illustrations/")
|
||
print(f"Added {len(new_catalog_items)} new items to catalog (total in catalog: {len(catalog)})")
|
||
|
||
# 4. Attach to external symbols
|
||
updated_ext_count = 0
|
||
for norm_title, ills in symbol_to_illustrations.items():
|
||
# Match in external symbols (exact or subkey)
|
||
target_keys = []
|
||
if norm_title in ext_symbols:
|
||
target_keys.append(norm_title)
|
||
else:
|
||
for k in ext_symbols:
|
||
if k == norm_title or norm_title.startswith(k + ' ') or k.startswith(norm_title + ' '):
|
||
target_keys.append(k)
|
||
|
||
for tk in target_keys:
|
||
existing_ills = ext_symbols[tk].get('illustrations', [])
|
||
existing_files = set(x.get('file') for x in existing_ills)
|
||
added = 0
|
||
for ill in ills:
|
||
if ill['file'] not in existing_files:
|
||
existing_ills.append(ill)
|
||
existing_files.add(ill['file'])
|
||
added += 1
|
||
if added > 0:
|
||
ext_symbols[tk]['illustrations'] = existing_ills
|
||
updated_ext_count += 1
|
||
|
||
print(f"Updated {updated_ext_count} external symbols with Williams illustrations")
|
||
|
||
# 5. Attach to Core 1000 passports
|
||
updated_passports_count = 0
|
||
for norm_title, ills in symbol_to_illustrations.items():
|
||
if norm_title in core_lookup:
|
||
pid = core_lookup[norm_title]
|
||
p_data = passports[pid]
|
||
existing_ills = p_data.get('illustrations', [])
|
||
existing_files = set(x.get('file') for x in existing_ills)
|
||
added = 0
|
||
for ill in ills:
|
||
if ill['file'] not in existing_files:
|
||
existing_ills.append(ill)
|
||
existing_files.add(ill['file'])
|
||
added += 1
|
||
if added > 0:
|
||
p_data['illustrations'] = existing_ills
|
||
updated_passports_count += 1
|
||
|
||
print(f"Updated {updated_passports_count} Core 1000 passports with Williams illustrations")
|
||
|
||
# Save everything back to disk
|
||
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(ext_symbols, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(STAGING_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||
|
||
print("=== Successfully finished Williams Master Enrichment! ===")
|
||
|
||
if __name__ == '__main__':
|
||
run()
|