import json
import re
from pathlib import Path

BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'

with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
    ext = json.load(f)

bad_keys = []
for k in ext.keys():
    # Check digits
    if any(c.isdigit() for c in k):
        bad_keys.append(k)
        continue
    # Check garbage phrases
    if any(phrase in k for phrase in [
        'дерево 7', 'см.', 'рис.', 'богатую символику', 'после обряда', 'вое',
        'на картине изображено', 'живописные', 'грецкий орех в данном',
        'жук-олень может в данном', 'в алхимии', 'в магии', 'в исламе',
        'у китайцев', 'у евреев', 'у христиан', 'у греков', 'у кельтов',
        'у маори', 'у бушменов', 'то же в', 'числа 3', 'числа 2', 'числа 1',
        'содержание дерево', 'предисловие к русскому'
    ]):
        bad_keys.append(k)
        continue
    # Check sentence fragments (>= 4 words that are not compound symbol names)
    words = k.split()
    if len(words) >= 4 and not any(k.startswith(p) for p in ['семь ', 'четыре ', 'пять ', 'восемь ', 'двенадцать ']):
        bad_keys.append(k)
        continue

print(f"Total bad keys identified for removal: {len(bad_keys)}")
for bk in bad_keys[:15]:
    print("  -", repr(bk))

# Remove bad keys
for bk in bad_keys:
    del ext[bk]

# Also remove Cooper / Kortunova / Timofey from sources in remaining keys so we can re-add them cleanly
for k, v in ext.items():
    v['sources'] = [s for s in v.get('sources', []) if s.get('author') not in ['Дж. Купер', 'Н.Д. Кортунова', 'Тимофей из Газы']]

with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
    json.dump(ext, f, ensure_ascii=False, indent=2)

# Update catalog
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
    cat = json.load(f)
cat = [c for c in cat if c.get('normalized') not in set(bad_keys)]
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
    json.dump(cat, f, ensure_ascii=False, indent=2)

print(f"Purge complete! Remaining clean external symbols: {len(ext)}")