219 lines
8.3 KiB
Python
219 lines
8.3 KiB
Python
import re
|
||
import json
|
||
import time
|
||
from pathlib import Path
|
||
import fitz
|
||
|
||
BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
|
||
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
|
||
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
|
||
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
|
||
COOPER_PDF = Path('/home/alexives/Desktop/newsources/Kuper_Dzh_Entsiklopedia_simvolov.pdf')
|
||
ILLUST_DIR = BASE_DIR / 'assets/illustrations'
|
||
|
||
t0 = time.time()
|
||
print("=" * 80)
|
||
print("📖 FAST COOPER INGESTION WITH HYPHENATION REPAIR")
|
||
print("=" * 80)
|
||
|
||
doc = fitz.open(str(COOPER_PDF))
|
||
print(f"Cooper PDF opened: {len(doc)} pages in {time.time()-t0:.2f}s")
|
||
|
||
# 1. First pass: extract illustrations and their captions
|
||
page_illustrations = {} # pno -> list of {file, caption, id, num}
|
||
for pno in range(6, len(doc)-1):
|
||
txt = doc[pno].get_text()
|
||
fig_matches = list(re.finditer(r'Рис\.\s*(\d+)[\.:]?\s*\n?([^\n]+(?:\n[^\n]+)?)', txt))
|
||
if not fig_matches:
|
||
continue
|
||
imgs = doc[pno].get_images()
|
||
if not imgs:
|
||
continue
|
||
|
||
extracted_here = []
|
||
for f_idx, fm in enumerate(fig_matches):
|
||
if f_idx < len(imgs):
|
||
f_num = fm.group(1)
|
||
raw_cap = fm.group(2).replace('\n', ' ').strip()
|
||
clean_cap = re.sub(r'\s+', ' ', raw_cap)[:130].strip()
|
||
xref = imgs[f_idx][0]
|
||
try:
|
||
base = doc.extract_image(xref)
|
||
w, h = base["width"], base["height"]
|
||
ext_fmt = base["ext"]
|
||
if w >= 140 and h >= 140:
|
||
fname = f"cooper_fig_{int(f_num):03d}_p{pno+1}.{ext_fmt}"
|
||
out_path = ILLUST_DIR / fname
|
||
if not out_path.exists():
|
||
with open(out_path, "wb") as f_out:
|
||
f_out.write(base["image"])
|
||
extracted_here.append({
|
||
"id": f"cooper_fig_{f_num}",
|
||
"file": f"assets/illustrations/{fname}",
|
||
"caption": f"Рис. {f_num}. {clean_cap}",
|
||
"page": pno + 1,
|
||
"width": w,
|
||
"height": h,
|
||
"source": "Дж. Купер · «Энциклопедия символов» (1995)"
|
||
})
|
||
except Exception:
|
||
pass
|
||
if extracted_here:
|
||
page_illustrations[pno] = extracted_here
|
||
|
||
print(f"Extracted/verified {sum(len(v) for v in page_illustrations.values())} Cooper illustrations in {time.time()-t0:.2f}s")
|
||
|
||
# 2. Second pass: Parse dictionary entries with de-hyphenation
|
||
def dehyphen(text: str) -> str:
|
||
# Fix soft hyphens (\xad), em-dashes, en-dashes at line breaks
|
||
return re.sub(r'([А-Яа-яA-Za-zЁё]+)[-\xad\u2010\u2011\u2012\u2013\u2014]\s*\n\s*([А-Яа-яA-Za-zЁё]+)', r'\1\2', text)
|
||
|
||
ru_alphabet = 'абвгдежзийклмнопрстуфхцчшщэюя'
|
||
cur_alpha_idx = 0
|
||
articles = {} # term -> {text, page, ills}
|
||
cur_term = None
|
||
cur_lines = []
|
||
cur_pno = 0
|
||
|
||
stop_terms = {
|
||
'энциклопедия символов', 'иллюстрации', 'содержание', 'указатель',
|
||
'латинские термины', 'библиография', 'введение', 'предисловие', 'рис'
|
||
}
|
||
|
||
for pno in range(6, len(doc)-1):
|
||
raw_txt = doc[pno].get_text()
|
||
clean_txt = dehyphen(raw_txt)
|
||
lines = [l.strip() for l in clean_txt.splitlines() if l.strip()]
|
||
|
||
for l in lines:
|
||
if l.isdigit() or len(l) <= 1:
|
||
continue
|
||
|
||
# Single letter heading
|
||
if len(l) == 1 and l.lower() in ru_alphabet:
|
||
l_idx = ru_alphabet.find(l.lower())
|
||
if l_idx >= cur_alpha_idx:
|
||
cur_alpha_idx = l_idx
|
||
continue
|
||
|
||
# Match entry headword: 'Word.' or 'Word -'
|
||
m = re.match(r'^([А-ЯЁ][а-яёА-ЯЁ\s\-]{1,30}?)(?:\.|\s+—|\s+-)\s*(.*)', l)
|
||
is_valid = False
|
||
if m:
|
||
cand = m.group(1).strip()
|
||
rest = m.group(2).strip()
|
||
norm = cand.lower()
|
||
if norm not in stop_terms and not any(c.isdigit() for c in norm) and len(norm.split()) <= 3:
|
||
first_ch = norm[0]
|
||
if first_ch in ru_alphabet:
|
||
ch_idx = ru_alphabet.find(first_ch)
|
||
# Enforce loose forward progression in alphabet
|
||
if ch_idx >= cur_alpha_idx and ch_idx <= cur_alpha_idx + 2:
|
||
cur_alpha_idx = ch_idx
|
||
is_valid = True
|
||
|
||
if is_valid:
|
||
if cur_term and cur_lines:
|
||
full_text = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
|
||
if len(full_text) >= 25:
|
||
norm_k = cur_term.lower()
|
||
if norm_k not in articles or len(full_text) > len(articles[norm_k]['text']):
|
||
articles[norm_k] = {
|
||
'term': cur_term,
|
||
'text': full_text,
|
||
'page': cur_pno,
|
||
'illustrations': page_illustrations.get(cur_pno, [])
|
||
}
|
||
cur_term = cand.capitalize()
|
||
cur_pno = pno
|
||
cur_lines = [rest] if rest else []
|
||
else:
|
||
if cur_term and not l.startswith('Рис.'):
|
||
cur_lines.append(l)
|
||
|
||
if cur_term and cur_lines:
|
||
full_text = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
|
||
if len(full_text) >= 25:
|
||
norm_k = cur_term.lower()
|
||
articles[norm_k] = {
|
||
'term': cur_term,
|
||
'text': full_text,
|
||
'page': cur_pno,
|
||
'illustrations': page_illustrations.get(cur_pno, [])
|
||
}
|
||
|
||
print(f"Extracted {len(articles)} clean Cooper articles in {time.time()-t0:.2f}s!")
|
||
|
||
# 3. Load database and insert clean Cooper entries
|
||
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
|
||
ext = json.load(f)
|
||
|
||
with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
|
||
passports = json.load(f)
|
||
|
||
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
|
||
catalog = json.load(f)
|
||
|
||
core_lookup = {}
|
||
for pid, p in passports.items():
|
||
w = p.get('word', '').strip().lower()
|
||
if w:
|
||
core_lookup[w] = pid
|
||
|
||
added_ext = 0
|
||
for norm_k, item in articles.items():
|
||
if norm_k not in ext:
|
||
ext[norm_k] = {
|
||
'term': item['term'].upper(),
|
||
'normalized': norm_k,
|
||
'category_guess': 'Академический универсальный символ',
|
||
'sources': [],
|
||
'illustrations': [],
|
||
'created_at': '2026-10-08T20:00:00',
|
||
'updated_at': '2026-10-08T20:00:00'
|
||
}
|
||
added_ext += 1
|
||
|
||
ext[norm_k]['sources'].append({
|
||
'author': 'Дж. Купер',
|
||
'work': '«Энциклопедия символов» (1995)',
|
||
'excerpt': item['text'][:350] + '…' if len(item['text']) > 350 else item['text'],
|
||
'full_text': item['text']
|
||
})
|
||
|
||
for ill in item['illustrations']:
|
||
if not any(x.get('file') == ill['file'] for x in ext[norm_k].get('illustrations', [])):
|
||
ext[norm_k]['illustrations'].append(ill)
|
||
catalog.append({
|
||
'id': ill['id'],
|
||
'file': ill['file'],
|
||
'symbol': item['term'].upper(),
|
||
'normalized': norm_k,
|
||
'caption': ill['caption'],
|
||
'source': ill['source'],
|
||
'page': ill['page'],
|
||
'width': ill['width'],
|
||
'height': ill['height'],
|
||
'is_core_1000': norm_k in core_lookup,
|
||
'core_id': core_lookup.get(norm_k)
|
||
})
|
||
|
||
if norm_k in core_lookup:
|
||
pid = core_lookup[norm_k]
|
||
p_item = passports[pid]
|
||
for ill in item['illustrations']:
|
||
if not any(x.get('file') == ill['file'] for x in p_item.get('illustrations', [])):
|
||
p_item['illustrations'].append(ill)
|
||
|
||
# Save files
|
||
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(ext, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"✅ Cooper finished in {time.time()-t0:.2f}s! Added {added_ext} new symbols, total symbols in ext: {len(ext)}")
|