import re
import json
import time
from pathlib import Path
import fitz

BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
CHUNAKOVA_PDF = BASE_DIR / 'dictionaries/Чунакова О.М. - Пехлевийский словарь зороастрийских терминов, мифических персонажей и мифологических символов. M. 2004.pdf'

t0 = time.time()
print("=" * 80)
print("🧹 DEEP CORPUS CLEANING & UNIVERSAL DE-HYPHENATION")
print("=" * 80)

# Load existing databases
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
    ext = json.load(f)

with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
    passports = json.load(f)

with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
    catalog = json.load(f)

print(f"Initial: {len(ext)} ext symbols, {len(passports)} passports, {len(catalog)} catalog entries")

# ------------------------------------------------------------------------------
# 1. PARSE CLEAN CHUNAKOVA ARTICLES & PURGE BLOATED/CORRUPTED ONES
# ------------------------------------------------------------------------------
print(">>> Step 1: Parsing clean Chunakova articles...")

doc_ch = fitz.open(str(CHUNAKOVA_PDF))

def dehyphen_simple(text: str) -> str:
    text = re.sub(r'([а-яёА-ЯЁA-Za-z]+)\xad\s*\n\s*([а-яёА-ЯЁA-Za-z]+)', r'\1\2', text)
    text = re.sub(r'([а-яёА-ЯЁA-Za-z]+)\xad\s+([а-яёА-ЯЁA-Za-z]+)', r'\1\2', text)
    text = text.replace('\xad', '')
    return text

bad_prefixes = ('его ', 'ее ', 'их ', 'этот ', 'эта ', 'эти ', 'определение ', 'эпитеты ', 'символ ', 'как ', 'в ', 'по ', 'на ', 'при ', 'от ', 'с ', 'для ', 'из ', 'три ', 'четыре ', 'пять ', 'шесть ', 'семь ', 'восемь ', 'девять ', 'десять ')

clean_chunakova = {}
cur_h = None
cur_lines = []

for p in range(12, 285):
    raw = doc_ch[p].get_text()
    cleaned = dehyphen_simple(raw)
    for line in cleaned.splitlines():
        line = line.strip()
        if not line or line.isdigit() or len(line) <= 2:
            continue
        m = re.match(r'^([А-ЯЁ][а-яёА-ЯЁ\s\-]{1,35}?)\s*[—–-]\s*(.*)', line)
        if m:
            head = m.group(1).strip()
            rest = m.group(2).strip()
            norm = head.lower()
            if not any(norm.startswith(bp) for bp in bad_prefixes) and not any(c.isdigit() for c in head):
                if cur_h and cur_lines:
                    txt = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
                    if len(txt) >= 25:
                        clean_chunakova[cur_h.lower()] = (cur_h, txt)
                cur_h = head
                cur_lines = [rest] if rest else []
                continue
        if cur_h:
            cur_lines.append(line)

if cur_h and cur_lines:
    txt = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
    if len(txt) >= 25:
        clean_chunakova[cur_h.lower()] = (cur_h, txt)

print(f"Extracted {len(clean_chunakova)} clean Chunakova articles in {time.time()-t0:.2f}s")

# Purge corrupted Chunakova entries from ext
chunakova_bloat = ['его символ', 'определение дахака', 'три сферы-стоянки', 'эпитеты ахримана']
for b_k in chunakova_bloat:
    if b_k in ext:
        del ext[b_k]

# Inject clean Chunakova articles
for norm_k, (term_c, text_c) in clean_chunakova.items():
    if norm_k in ext:
        # Update Chunakova source
        sources = [s for s in ext[norm_k].get('sources', []) if 'Чунакова' not in s.get('author', '')]
        sources.append({
            'author': 'О.М. Чунакова',
            'work': '«Пехлевийский словарь зороастрийских терминов»',
            'excerpt': (text_c[:250] + '…') if len(text_c) > 250 else text_c,
            'full_text': text_c
        })
        ext[norm_k]['sources'] = sources
    else:
        # Create concise entry if valid length
        if len(text_c) >= 30:
            ext[norm_k] = {
                'term': term_c,
                'normalized': norm_k,
                'category_guess': 'Зороастрийская сакральная традиция',
                'sources': [{
                    'author': 'О.М. Чунакова',
                    'work': '«Пехлевийский словарь зороастрийских терминов»',
                    'excerpt': (text_c[:250] + '…') if len(text_c) > 250 else text_c,
                    'full_text': text_c
                }],
                'illustrations': []
            }

print(">>> Step 1 complete.")

# ------------------------------------------------------------------------------
# 2. PURGE GHOST EMPTY KEYS (97 KEYS) & MOVE VALUABLE ILLUSTRATIONS
# ------------------------------------------------------------------------------
print(">>> Step 2: Purging empty ghost keys and saving illustrations...")

# Move valuable illustrations to legitimate targets
ill_relocations = {
    'огонь элемент': 'огонь',
    'сердце символизируется лотосом': 'сердце',
    'корзина с фруктами': 'фрукты',
    'большие рыбы': 'рыба',
    'фламандские пословицы': 'пословицы',
    'меняла с женой': 'меняла',
    'аллегории пяти чувств': 'пять чувств'
}

for ghost_k, target_k in ill_relocations.items():
    if ghost_k in ext and target_k in ext:
        for ill in ext[ghost_k].get('illustrations', []):
            if not any(x.get('file') == ill.get('file') for x in ext[target_k].get('illustrations', [])):
                ext[target_k].setdefault('illustrations', []).append(ill)

# Find and delete all items with 0 sources
ghosts_deleted = 0
for k in list(ext.keys()):
    if not ext[k].get('sources'):
        del ext[k]
        ghosts_deleted += 1

print(f">>> Step 2 complete: Purged {ghosts_deleted} ghost keys with 0 sources.")

# ------------------------------------------------------------------------------
# 3. UNIVERSAL DE-HYPHENATION ENGINE & ARTIFACT REPAIR
# ------------------------------------------------------------------------------
print(">>> Step 3: Running universal de-hyphenation engine across all texts...")

compound_prefixes = {
    'северо', 'юго', 'западно', 'восточно', 'темно', 'светло', 'ярко', 'бледно',
    'военно', 'научно', 'социально', 'политико', 'физико', 'химико', 'историко',
    'литературно', 'торгово', 'русско', 'греко', 'англо', 'франко', 'индо', 'китае',
    'евро', 'афро', 'арабо', 'мусульмано', 'христиано', 'иудео', 'буддо', 'инь', 'ян'
}

particles = {'то', 'либо', 'нибудь', 'таки'}

def universal_clean_text(text: str) -> str:
    if not text or not isinstance(text, str):
        return text

    # 1. Embedded footnotes / OCR captions inside words
    text = re.sub(r'грече1\s+П\s+о\s+р\s+о\s+к[^\.\n]*?ского', 'греческого', text)
    text = re.sub(r'по1\s+Г\s+о\s+р\s+г\s+у\s+л\s+ь\s+я[^\.\n]*?сланников', 'посланников', text)
    text = re.sub(r'улы1\s+Вспомним[^\.\n]*?бающаяся', 'улыбающаяся', text)
    text = re.sub(r'вер393\s+Сирена[^\.\n]*?сии', 'версии', text)
    text = re.sub(r'отдыха141\s+Дерево[^\.\n]*?древо', 'отдыха', text)
    text = re.sub(r'Восемь благоприятных символов\s+23\s+Будды', 'Восемь благоприятных символов Будды', text)
    text = re.sub(r'1\s+С\s+у\s+з\s+н\s+и\s+ц\s+а\s*\.[^\.\n]*?э\s+л\s+е\s+м\s*', '', text)

    # 2. Cut off trailing index catalog in 'ястреб-перепелятник'
    if 'Птицы; Хаос; Ребенок; Колонны' in text:
        text = text.split('Птицы; Хаос; Ребенок')[0].strip()

    # 3. Soft hyphens (\xad)
    text = re.sub(r'([а-яёА-ЯЁA-Za-z]{2,})\xad\s+([а-яёА-ЯЁA-Za-z]{2,})', r'\1\2', text)
    text = text.replace('\xad', '')

    # 4. Newline broken hyphens
    def repl_nl(m):
        w1, w2 = m.group(1), m.group(2)
        if w1.lower() in compound_prefixes:
            return f'{w1}-{w2}'
        if w1.lower().endswith(('о', 'е')) and w2.lower() in ('и', 'или'):
            return f'{w1}- {w2}'
        return f'{w1}{w2}'
    text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s*\n\s*([а-яё]{2,})', repl_nl, text)

    # 5. In-line broken hyphenations: 'сло- во'
    def repl_hyphen_space(m):
        w1, w2 = m.group(1), m.group(2)
        w1_low = w1.lower()
        w2_low = w2.lower()
        # Hanging hyphen: 'научно- и'
        if w1_low.endswith(('о', 'е')) and w2_low in ('и', 'или'):
            return f'{w1}- {w2}'
        # Compound prefix: 'северо- восточный' -> 'северо-восточный'
        if w1_low in compound_prefixes:
            return f'{w1}-{w2}'
        # Particles: 'какой- то' -> 'какой-то'
        if w2_low in particles:
            return f'{w1}-{w2}'
        if w1_low in ('из', 'по') and w2_low in ('за', 'под', 'прежнему', 'новому', 'видимому'):
            return f'{w1}-{w2}'
        # Default: fuse word split
        return f'{w1}{w2}'

    # Double pass for chained hyphenations
    text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)
    text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)

    # 6. Spacing before punctuation
    text = re.sub(r'\s+([,.:;?!])', r'\1', text)
    text = re.sub(r'\(\s+', '(', text)
    text = re.sub(r'\s+\)', ')', text)

    # 7. Normalize whitespace
    text = re.sub(r'[ \t]{2,}', ' ', text)
    return text.strip()

def clean_data_structure(data):
    """Recursively clean all strings in dicts/lists."""
    if isinstance(data, dict):
        return {k: clean_data_structure(v) for k, v in data.items()}
    elif isinstance(data, list):
        return [clean_data_structure(item) for item in data]
    elif isinstance(data, str):
        return universal_clean_text(data)
    return data

ext = clean_data_structure(ext)
passports = clean_data_structure(passports)
catalog = clean_data_structure(catalog)

print(">>> Step 3 complete: All strings de-hyphenated and cleaned.")

# ------------------------------------------------------------------------------
# 4. FINAL SOURCE DEDUPLICATION
# ------------------------------------------------------------------------------
print(">>> Step 4: Final source deduplication...")

dedup_count = 0
for k, item in ext.items():
    sources = item.get('sources', [])
    if not sources:
        continue
    unique = []
    seen = set()
    for s in sources:
        author = s.get('author', '').strip().lower()
        work = s.get('work', '').strip().lower()
        snippet = s.get('full_text', s.get('excerpt', ''))[:50].strip().lower()
        sig = (author, work, snippet)
        if sig not in seen:
            seen.add(sig)
            unique.append(s)
        else:
            dedup_count += 1
    item['sources'] = unique

print(f">>> Step 4 complete: Deduplicated {dedup_count} sources.")

# ------------------------------------------------------------------------------
# 5. SAVE ALL DATABASES
# ------------------------------------------------------------------------------
print(">>> Step 5: Saving databases...")

with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
    json.dump(ext, f, ensure_ascii=False, indent=2)

with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
    json.dump(passports, f, ensure_ascii=False, indent=2)

with open(STAGING_PATH, 'w', encoding='utf-8') as f:
    json.dump(passports, f, ensure_ascii=False, indent=2)

with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
    json.dump(catalog, f, ensure_ascii=False, indent=2)

print("=" * 80)
print(f"✅ DEEP CLEANING COMPLETE IN {time.time()-t0:.2f}s!")
print(f"Ext symbols: {len(ext)}")
print(f"Passports: {len(passports)}")
print(f"Catalog entries: {len(catalog)}")
print("=" * 80)