import re
import json
import time
from pathlib import Path
import fitz

BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
COOPER_PDF = Path('/home/alexives/Desktop/newsources/Kuper_Dzh_Entsiklopedia_simvolov.pdf')
ILLUST_DIR = BASE_DIR / 'assets/illustrations'

t0 = time.time()
print("=" * 80)
print("📖 FAST COOPER INGESTION WITH HYPHENATION REPAIR")
print("=" * 80)

doc = fitz.open(str(COOPER_PDF))
print(f"Cooper PDF opened: {len(doc)} pages in {time.time()-t0:.2f}s")

# 1. First pass: extract illustrations and their captions
page_illustrations = {} # pno -> list of {file, caption, id, num}
for pno in range(6, len(doc)-1):
    txt = doc[pno].get_text()
    fig_matches = list(re.finditer(r'Рис\.\s*(\d+)[\.:]?\s*\n?([^\n]+(?:\n[^\n]+)?)', txt))
    if not fig_matches:
        continue
    imgs = doc[pno].get_images()
    if not imgs:
        continue

    extracted_here = []
    for f_idx, fm in enumerate(fig_matches):
        if f_idx < len(imgs):
            f_num = fm.group(1)
            raw_cap = fm.group(2).replace('\n', ' ').strip()
            clean_cap = re.sub(r'\s+', ' ', raw_cap)[:130].strip()
            xref = imgs[f_idx][0]
            try:
                base = doc.extract_image(xref)
                w, h = base["width"], base["height"]
                ext_fmt = base["ext"]
                if w >= 140 and h >= 140:
                    fname = f"cooper_fig_{int(f_num):03d}_p{pno+1}.{ext_fmt}"
                    out_path = ILLUST_DIR / fname
                    if not out_path.exists():
                        with open(out_path, "wb") as f_out:
                            f_out.write(base["image"])
                    extracted_here.append({
                        "id": f"cooper_fig_{f_num}",
                        "file": f"assets/illustrations/{fname}",
                        "caption": f"Рис. {f_num}. {clean_cap}",
                        "page": pno + 1,
                        "width": w,
                        "height": h,
                        "source": "Дж. Купер · «Энциклопедия символов» (1995)"
                    })
            except Exception:
                pass
    if extracted_here:
        page_illustrations[pno] = extracted_here

print(f"Extracted/verified {sum(len(v) for v in page_illustrations.values())} Cooper illustrations in {time.time()-t0:.2f}s")

# 2. Second pass: Parse dictionary entries with de-hyphenation
def dehyphen(text: str) -> str:
    # Fix soft hyphens (\xad), em-dashes, en-dashes at line breaks
    return re.sub(r'([А-Яа-яA-Za-zЁё]+)[-\xad\u2010\u2011\u2012\u2013\u2014]\s*\n\s*([А-Яа-яA-Za-zЁё]+)', r'\1\2', text)

ru_alphabet = 'абвгдежзийклмнопрстуфхцчшщэюя'
cur_alpha_idx = 0
articles = {} # term -> {text, page, ills}
cur_term = None
cur_lines = []
cur_pno = 0

stop_terms = {
    'энциклопедия символов', 'иллюстрации', 'содержание', 'указатель',
    'латинские термины', 'библиография', 'введение', 'предисловие', 'рис'
}

for pno in range(6, len(doc)-1):
    raw_txt = doc[pno].get_text()
    clean_txt = dehyphen(raw_txt)
    lines = [l.strip() for l in clean_txt.splitlines() if l.strip()]

    for l in lines:
        if l.isdigit() or len(l) <= 1:
            continue
        
        # Single letter heading
        if len(l) == 1 and l.lower() in ru_alphabet:
            l_idx = ru_alphabet.find(l.lower())
            if l_idx >= cur_alpha_idx:
                cur_alpha_idx = l_idx
            continue

        # Match entry headword: 'Word.' or 'Word -'
        m = re.match(r'^([А-ЯЁ][а-яёА-ЯЁ\s\-]{1,30}?)(?:\.|\s+—|\s+-)\s*(.*)', l)
        is_valid = False
        if m:
            cand = m.group(1).strip()
            rest = m.group(2).strip()
            norm = cand.lower()
            if norm not in stop_terms and not any(c.isdigit() for c in norm) and len(norm.split()) <= 3:
                first_ch = norm[0]
                if first_ch in ru_alphabet:
                    ch_idx = ru_alphabet.find(first_ch)
                    # Enforce loose forward progression in alphabet
                    if ch_idx >= cur_alpha_idx and ch_idx <= cur_alpha_idx + 2:
                        cur_alpha_idx = ch_idx
                        is_valid = True

        if is_valid:
            if cur_term and cur_lines:
                full_text = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
                if len(full_text) >= 25:
                    norm_k = cur_term.lower()
                    if norm_k not in articles or len(full_text) > len(articles[norm_k]['text']):
                        articles[norm_k] = {
                            'term': cur_term,
                            'text': full_text,
                            'page': cur_pno,
                            'illustrations': page_illustrations.get(cur_pno, [])
                        }
            cur_term = cand.capitalize()
            cur_pno = pno
            cur_lines = [rest] if rest else []
        else:
            if cur_term and not l.startswith('Рис.'):
                cur_lines.append(l)

if cur_term and cur_lines:
    full_text = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
    if len(full_text) >= 25:
        norm_k = cur_term.lower()
        articles[norm_k] = {
            'term': cur_term,
            'text': full_text,
            'page': cur_pno,
            'illustrations': page_illustrations.get(cur_pno, [])
        }

print(f"Extracted {len(articles)} clean Cooper articles in {time.time()-t0:.2f}s!")

# 3. Load database and insert clean Cooper entries
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
    ext = json.load(f)

with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
    passports = json.load(f)

with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
    catalog = json.load(f)

core_lookup = {}
for pid, p in passports.items():
    w = p.get('word', '').strip().lower()
    if w:
        core_lookup[w] = pid

added_ext = 0
for norm_k, item in articles.items():
    if norm_k not in ext:
        ext[norm_k] = {
            'term': item['term'].upper(),
            'normalized': norm_k,
            'category_guess': 'Академический универсальный символ',
            'sources': [],
            'illustrations': [],
            'created_at': '2026-10-08T20:00:00',
            'updated_at': '2026-10-08T20:00:00'
        }
        added_ext += 1

    ext[norm_k]['sources'].append({
        'author': 'Дж. Купер',
        'work': '«Энциклопедия символов» (1995)',
        'excerpt': item['text'][:350] + '…' if len(item['text']) > 350 else item['text'],
        'full_text': item['text']
    })

    for ill in item['illustrations']:
        if not any(x.get('file') == ill['file'] for x in ext[norm_k].get('illustrations', [])):
            ext[norm_k]['illustrations'].append(ill)
            catalog.append({
                'id': ill['id'],
                'file': ill['file'],
                'symbol': item['term'].upper(),
                'normalized': norm_k,
                'caption': ill['caption'],
                'source': ill['source'],
                'page': ill['page'],
                'width': ill['width'],
                'height': ill['height'],
                'is_core_1000': norm_k in core_lookup,
                'core_id': core_lookup.get(norm_k)
            })

    if norm_k in core_lookup:
        pid = core_lookup[norm_k]
        p_item = passports[pid]
        for ill in item['illustrations']:
            if not any(x.get('file') == ill['file'] for x in p_item.get('illustrations', [])):
                p_item['illustrations'].append(ill)

# Save files
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
    json.dump(ext, f, ensure_ascii=False, indent=2)

with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
    json.dump(passports, f, ensure_ascii=False, indent=2)

with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
    json.dump(catalog, f, ensure_ascii=False, indent=2)

print(f"✅ Cooper finished in {time.time()-t0:.2f}s! Added {added_ext} new symbols, total symbols in ext: {len(ext)}")