feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets
This commit is contained in:
@@ -0,0 +1,230 @@
|
||||
import re
|
||||
import json
|
||||
from pathlib import Path
|
||||
import fitz
|
||||
|
||||
BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
|
||||
ILLUST_DIR = BASE_DIR / 'assets/illustrations'
|
||||
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
|
||||
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
|
||||
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
|
||||
STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
|
||||
PDF_PATH = Path('/home/alexives/Desktop/newsources/Уильямс Ч. - Китайская культура. Мифы, герои, символы - 2011.pdf')
|
||||
|
||||
ru_to_en = {
|
||||
'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo',
|
||||
'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm',
|
||||
'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u',
|
||||
'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'shch',
|
||||
'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya'
|
||||
}
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
slug = []
|
||||
for c in text.lower():
|
||||
if c in ru_to_en:
|
||||
slug.append(ru_to_en[c])
|
||||
elif c.isalnum():
|
||||
slug.append(c)
|
||||
elif c in [' ', '-', '_']:
|
||||
slug.append('_')
|
||||
res = re.sub(r'_+', '_', ''.join(slug)).strip('_')
|
||||
return res[:30] if res else 'symbol'
|
||||
|
||||
def run():
|
||||
print("=== Master Williams Illustration Extractor & Linker ===")
|
||||
if not PDF_PATH.exists():
|
||||
print(f"Error: PDF not found at {PDF_PATH}")
|
||||
return
|
||||
|
||||
doc = fitz.open(str(PDF_PATH))
|
||||
print(f"Opened Williams PDF: {len(doc)} pages")
|
||||
|
||||
# 1. Parse all articles with page ranges
|
||||
# Collect all headers and pages
|
||||
article_spans = [] # (title, start_p, end_p)
|
||||
all_headers = []
|
||||
|
||||
for p in range(12, 470):
|
||||
txt = doc[p].get_text()
|
||||
lines = [l.strip() for l in txt.splitlines() if l.strip()]
|
||||
for l in lines:
|
||||
if l.isupper() and 2 <= len(l) <= 35 and re.match(r'^[А-ЯЁ\s\-]+$', l):
|
||||
if not any(skip in l for skip in ['КИТАЙСКАЯ КУЛЬТУРА', 'СОДЕРЖАНИЕ', 'ВВЕДЕНИЕ', 'УКАЗАТЕЛЬ', 'ЧАСТЬ', 'ПРИЛОЖЕНИЕ']):
|
||||
all_headers.append((p, l))
|
||||
|
||||
# De-duplicate consecutive headers on same page
|
||||
filtered_headers = []
|
||||
for p, h in all_headers:
|
||||
if not filtered_headers or filtered_headers[-1][1] != h:
|
||||
filtered_headers.append((p, h))
|
||||
|
||||
print(f"Found {len(filtered_headers)} article headers in Williams")
|
||||
|
||||
# Map each page to the active article header
|
||||
page_to_article = {}
|
||||
for i, (p, h) in enumerate(filtered_headers):
|
||||
next_p = filtered_headers[i+1][0] if i+1 < len(filtered_headers) else 470
|
||||
for pg in range(p, next_p + 1):
|
||||
if pg not in page_to_article:
|
||||
page_to_article[pg] = h
|
||||
|
||||
# 2. Load existing catalog and external symbols
|
||||
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
|
||||
catalog = json.load(f)
|
||||
existing_catalog_files = set(x.get('file') for x in catalog)
|
||||
|
||||
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
|
||||
ext_symbols = json.load(f)
|
||||
|
||||
with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
|
||||
passports = json.load(f)
|
||||
|
||||
# Build core word lookup: normalized_word -> passport_id
|
||||
core_lookup = {}
|
||||
for pid, pdata in passports.items():
|
||||
w = pdata.get('word', '').strip().lower()
|
||||
if w:
|
||||
core_lookup[w] = pid
|
||||
|
||||
new_extracted_count = 0
|
||||
new_catalog_items = []
|
||||
symbol_to_illustrations = {} # normalized_symbol -> list of ill dicts
|
||||
|
||||
# 3. Extract images across all pages
|
||||
for p in range(12, 470):
|
||||
active_header = page_to_article.get(p)
|
||||
if not active_header:
|
||||
continue
|
||||
|
||||
slug = slugify(active_header)
|
||||
norm_title = active_header.lower()
|
||||
imgs = doc[p].get_images()
|
||||
if not imgs:
|
||||
continue
|
||||
|
||||
for idx, img_info in enumerate(imgs):
|
||||
xref = img_info[0]
|
||||
try:
|
||||
base_img = doc.extract_image(xref)
|
||||
w, h = base_img["width"], base_img["height"]
|
||||
ext = base_img["ext"]
|
||||
|
||||
# Extract both large illustrations (woodcuts) and medium glyphs/seals
|
||||
is_large_art = (w * h >= 20000) or (max(w, h) >= 160 and min(w, h) >= 90)
|
||||
is_sacred_glyph = (w >= 70 and h >= 70 and w * h >= 5000)
|
||||
|
||||
if not (is_large_art or is_sacred_glyph):
|
||||
continue
|
||||
|
||||
fname = f"wil_{p:03d}_{slug}_{idx}.{ext}"
|
||||
rel_path = f"assets/illustrations/{fname}"
|
||||
abs_path = ILLUST_DIR / fname
|
||||
|
||||
if not abs_path.exists():
|
||||
with open(abs_path, 'wb') as f_out:
|
||||
f_out.write(base_img['image'])
|
||||
new_extracted_count += 1
|
||||
|
||||
ill_id = f"wil_{p}_{idx}"
|
||||
caption_type = "Гравюра и образ" if is_large_art else "Сакральное начертание / Глиф"
|
||||
caption = f"Ч. Уильямс · {active_header.title()} ({caption_type})"
|
||||
|
||||
is_core = norm_title in core_lookup
|
||||
core_id = core_lookup.get(norm_title)
|
||||
|
||||
ill_entry = {
|
||||
"id": ill_id,
|
||||
"file": rel_path,
|
||||
"symbol": active_header,
|
||||
"normalized": norm_title,
|
||||
"caption": caption,
|
||||
"source": "Ч. Уильямс · «Китайская культура: символы и мифы» (2011)",
|
||||
"page": p + 1,
|
||||
"width": w,
|
||||
"height": h,
|
||||
"is_core_1000": is_core,
|
||||
"core_id": str(core_id) if core_id else None
|
||||
}
|
||||
|
||||
if rel_path not in existing_catalog_files:
|
||||
catalog.append(ill_entry)
|
||||
existing_catalog_files.add(rel_path)
|
||||
new_catalog_items.append(ill_entry)
|
||||
|
||||
if norm_title not in symbol_to_illustrations:
|
||||
symbol_to_illustrations[norm_title] = []
|
||||
# Avoid duplicate files for same symbol
|
||||
if not any(x['file'] == rel_path for x in symbol_to_illustrations[norm_title]):
|
||||
symbol_to_illustrations[norm_title].append(ill_entry)
|
||||
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
print(f"Extracted {new_extracted_count} new images to assets/illustrations/")
|
||||
print(f"Added {len(new_catalog_items)} new items to catalog (total in catalog: {len(catalog)})")
|
||||
|
||||
# 4. Attach to external symbols
|
||||
updated_ext_count = 0
|
||||
for norm_title, ills in symbol_to_illustrations.items():
|
||||
# Match in external symbols (exact or subkey)
|
||||
target_keys = []
|
||||
if norm_title in ext_symbols:
|
||||
target_keys.append(norm_title)
|
||||
else:
|
||||
for k in ext_symbols:
|
||||
if k == norm_title or norm_title.startswith(k + ' ') or k.startswith(norm_title + ' '):
|
||||
target_keys.append(k)
|
||||
|
||||
for tk in target_keys:
|
||||
existing_ills = ext_symbols[tk].get('illustrations', [])
|
||||
existing_files = set(x.get('file') for x in existing_ills)
|
||||
added = 0
|
||||
for ill in ills:
|
||||
if ill['file'] not in existing_files:
|
||||
existing_ills.append(ill)
|
||||
existing_files.add(ill['file'])
|
||||
added += 1
|
||||
if added > 0:
|
||||
ext_symbols[tk]['illustrations'] = existing_ills
|
||||
updated_ext_count += 1
|
||||
|
||||
print(f"Updated {updated_ext_count} external symbols with Williams illustrations")
|
||||
|
||||
# 5. Attach to Core 1000 passports
|
||||
updated_passports_count = 0
|
||||
for norm_title, ills in symbol_to_illustrations.items():
|
||||
if norm_title in core_lookup:
|
||||
pid = core_lookup[norm_title]
|
||||
p_data = passports[pid]
|
||||
existing_ills = p_data.get('illustrations', [])
|
||||
existing_files = set(x.get('file') for x in existing_ills)
|
||||
added = 0
|
||||
for ill in ills:
|
||||
if ill['file'] not in existing_files:
|
||||
existing_ills.append(ill)
|
||||
existing_files.add(ill['file'])
|
||||
added += 1
|
||||
if added > 0:
|
||||
p_data['illustrations'] = existing_ills
|
||||
updated_passports_count += 1
|
||||
|
||||
print(f"Updated {updated_passports_count} Core 1000 passports with Williams illustrations")
|
||||
|
||||
# Save everything back to disk
|
||||
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(ext_symbols, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(STAGING_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print("=== Successfully finished Williams Master Enrichment! ===")
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
Reference in New Issue
Block a user