feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
import fitz
|
||||
|
||||
BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
|
||||
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
|
||||
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
|
||||
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
|
||||
COOPER_PDF = Path('/home/alexives/Desktop/newsources/Kuper_Dzh_Entsiklopedia_simvolov.pdf')
|
||||
ILLUST_DIR = BASE_DIR / 'assets/illustrations'
|
||||
|
||||
t0 = time.time()
|
||||
print("=" * 80)
|
||||
print("📖 FAST COOPER INGESTION WITH HYPHENATION REPAIR")
|
||||
print("=" * 80)
|
||||
|
||||
doc = fitz.open(str(COOPER_PDF))
|
||||
print(f"Cooper PDF opened: {len(doc)} pages in {time.time()-t0:.2f}s")
|
||||
|
||||
# 1. First pass: extract illustrations and their captions
|
||||
page_illustrations = {} # pno -> list of {file, caption, id, num}
|
||||
for pno in range(6, len(doc)-1):
|
||||
txt = doc[pno].get_text()
|
||||
fig_matches = list(re.finditer(r'Рис\.\s*(\d+)[\.:]?\s*\n?([^\n]+(?:\n[^\n]+)?)', txt))
|
||||
if not fig_matches:
|
||||
continue
|
||||
imgs = doc[pno].get_images()
|
||||
if not imgs:
|
||||
continue
|
||||
|
||||
extracted_here = []
|
||||
for f_idx, fm in enumerate(fig_matches):
|
||||
if f_idx < len(imgs):
|
||||
f_num = fm.group(1)
|
||||
raw_cap = fm.group(2).replace('\n', ' ').strip()
|
||||
clean_cap = re.sub(r'\s+', ' ', raw_cap)[:130].strip()
|
||||
xref = imgs[f_idx][0]
|
||||
try:
|
||||
base = doc.extract_image(xref)
|
||||
w, h = base["width"], base["height"]
|
||||
ext_fmt = base["ext"]
|
||||
if w >= 140 and h >= 140:
|
||||
fname = f"cooper_fig_{int(f_num):03d}_p{pno+1}.{ext_fmt}"
|
||||
out_path = ILLUST_DIR / fname
|
||||
if not out_path.exists():
|
||||
with open(out_path, "wb") as f_out:
|
||||
f_out.write(base["image"])
|
||||
extracted_here.append({
|
||||
"id": f"cooper_fig_{f_num}",
|
||||
"file": f"assets/illustrations/{fname}",
|
||||
"caption": f"Рис. {f_num}. {clean_cap}",
|
||||
"page": pno + 1,
|
||||
"width": w,
|
||||
"height": h,
|
||||
"source": "Дж. Купер · «Энциклопедия символов» (1995)"
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
if extracted_here:
|
||||
page_illustrations[pno] = extracted_here
|
||||
|
||||
print(f"Extracted/verified {sum(len(v) for v in page_illustrations.values())} Cooper illustrations in {time.time()-t0:.2f}s")
|
||||
|
||||
# 2. Second pass: Parse dictionary entries with de-hyphenation
|
||||
def dehyphen(text: str) -> str:
|
||||
# Fix soft hyphens (\xad), em-dashes, en-dashes at line breaks
|
||||
return re.sub(r'([А-Яа-яA-Za-zЁё]+)[-\xad\u2010\u2011\u2012\u2013\u2014]\s*\n\s*([А-Яа-яA-Za-zЁё]+)', r'\1\2', text)
|
||||
|
||||
ru_alphabet = 'абвгдежзийклмнопрстуфхцчшщэюя'
|
||||
cur_alpha_idx = 0
|
||||
articles = {} # term -> {text, page, ills}
|
||||
cur_term = None
|
||||
cur_lines = []
|
||||
cur_pno = 0
|
||||
|
||||
stop_terms = {
|
||||
'энциклопедия символов', 'иллюстрации', 'содержание', 'указатель',
|
||||
'латинские термины', 'библиография', 'введение', 'предисловие', 'рис'
|
||||
}
|
||||
|
||||
for pno in range(6, len(doc)-1):
|
||||
raw_txt = doc[pno].get_text()
|
||||
clean_txt = dehyphen(raw_txt)
|
||||
lines = [l.strip() for l in clean_txt.splitlines() if l.strip()]
|
||||
|
||||
for l in lines:
|
||||
if l.isdigit() or len(l) <= 1:
|
||||
continue
|
||||
|
||||
# Single letter heading
|
||||
if len(l) == 1 and l.lower() in ru_alphabet:
|
||||
l_idx = ru_alphabet.find(l.lower())
|
||||
if l_idx >= cur_alpha_idx:
|
||||
cur_alpha_idx = l_idx
|
||||
continue
|
||||
|
||||
# Match entry headword: 'Word.' or 'Word -'
|
||||
m = re.match(r'^([А-ЯЁ][а-яёА-ЯЁ\s\-]{1,30}?)(?:\.|\s+—|\s+-)\s*(.*)', l)
|
||||
is_valid = False
|
||||
if m:
|
||||
cand = m.group(1).strip()
|
||||
rest = m.group(2).strip()
|
||||
norm = cand.lower()
|
||||
if norm not in stop_terms and not any(c.isdigit() for c in norm) and len(norm.split()) <= 3:
|
||||
first_ch = norm[0]
|
||||
if first_ch in ru_alphabet:
|
||||
ch_idx = ru_alphabet.find(first_ch)
|
||||
# Enforce loose forward progression in alphabet
|
||||
if ch_idx >= cur_alpha_idx and ch_idx <= cur_alpha_idx + 2:
|
||||
cur_alpha_idx = ch_idx
|
||||
is_valid = True
|
||||
|
||||
if is_valid:
|
||||
if cur_term and cur_lines:
|
||||
full_text = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
|
||||
if len(full_text) >= 25:
|
||||
norm_k = cur_term.lower()
|
||||
if norm_k not in articles or len(full_text) > len(articles[norm_k]['text']):
|
||||
articles[norm_k] = {
|
||||
'term': cur_term,
|
||||
'text': full_text,
|
||||
'page': cur_pno,
|
||||
'illustrations': page_illustrations.get(cur_pno, [])
|
||||
}
|
||||
cur_term = cand.capitalize()
|
||||
cur_pno = pno
|
||||
cur_lines = [rest] if rest else []
|
||||
else:
|
||||
if cur_term and not l.startswith('Рис.'):
|
||||
cur_lines.append(l)
|
||||
|
||||
if cur_term and cur_lines:
|
||||
full_text = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
|
||||
if len(full_text) >= 25:
|
||||
norm_k = cur_term.lower()
|
||||
articles[norm_k] = {
|
||||
'term': cur_term,
|
||||
'text': full_text,
|
||||
'page': cur_pno,
|
||||
'illustrations': page_illustrations.get(cur_pno, [])
|
||||
}
|
||||
|
||||
print(f"Extracted {len(articles)} clean Cooper articles in {time.time()-t0:.2f}s!")
|
||||
|
||||
# 3. Load database and insert clean Cooper entries
|
||||
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
|
||||
ext = json.load(f)
|
||||
|
||||
with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
|
||||
passports = json.load(f)
|
||||
|
||||
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
|
||||
catalog = json.load(f)
|
||||
|
||||
core_lookup = {}
|
||||
for pid, p in passports.items():
|
||||
w = p.get('word', '').strip().lower()
|
||||
if w:
|
||||
core_lookup[w] = pid
|
||||
|
||||
added_ext = 0
|
||||
for norm_k, item in articles.items():
|
||||
if norm_k not in ext:
|
||||
ext[norm_k] = {
|
||||
'term': item['term'].upper(),
|
||||
'normalized': norm_k,
|
||||
'category_guess': 'Академический универсальный символ',
|
||||
'sources': [],
|
||||
'illustrations': [],
|
||||
'created_at': '2026-10-08T20:00:00',
|
||||
'updated_at': '2026-10-08T20:00:00'
|
||||
}
|
||||
added_ext += 1
|
||||
|
||||
ext[norm_k]['sources'].append({
|
||||
'author': 'Дж. Купер',
|
||||
'work': '«Энциклопедия символов» (1995)',
|
||||
'excerpt': item['text'][:350] + '…' if len(item['text']) > 350 else item['text'],
|
||||
'full_text': item['text']
|
||||
})
|
||||
|
||||
for ill in item['illustrations']:
|
||||
if not any(x.get('file') == ill['file'] for x in ext[norm_k].get('illustrations', [])):
|
||||
ext[norm_k]['illustrations'].append(ill)
|
||||
catalog.append({
|
||||
'id': ill['id'],
|
||||
'file': ill['file'],
|
||||
'symbol': item['term'].upper(),
|
||||
'normalized': norm_k,
|
||||
'caption': ill['caption'],
|
||||
'source': ill['source'],
|
||||
'page': ill['page'],
|
||||
'width': ill['width'],
|
||||
'height': ill['height'],
|
||||
'is_core_1000': norm_k in core_lookup,
|
||||
'core_id': core_lookup.get(norm_k)
|
||||
})
|
||||
|
||||
if norm_k in core_lookup:
|
||||
pid = core_lookup[norm_k]
|
||||
p_item = passports[pid]
|
||||
for ill in item['illustrations']:
|
||||
if not any(x.get('file') == ill['file'] for x in p_item.get('illustrations', [])):
|
||||
p_item['illustrations'].append(ill)
|
||||
|
||||
# Save files
|
||||
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(ext, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
|
||||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"✅ Cooper finished in {time.time()-t0:.2f}s! Added {added_ext} new symbols, total symbols in ext: {len(ext)}")
|
||||
Reference in New Issue
Block a user