289 lines
12 KiB
Python
289 lines
12 KiB
Python
import re
|
||
import json
|
||
import time
|
||
from pathlib import Path
|
||
import fitz
|
||
|
||
BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
|
||
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
|
||
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
|
||
STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
|
||
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
|
||
CHUNAKOVA_PDF = BASE_DIR / 'dictionaries/Чунакова О.М. - Пехлевийский словарь зороастрийских терминов, мифических персонажей и мифологических символов. M. 2004.pdf'
|
||
|
||
t0 = time.time()
|
||
print("=" * 80)
|
||
print("🧹 DEEP CORPUS CLEANING & UNIVERSAL DE-HYPHENATION")
|
||
print("=" * 80)
|
||
|
||
# Load existing databases
|
||
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
|
||
ext = json.load(f)
|
||
|
||
with open(PASSPORTS_PATH, 'r', encoding='utf-8') as f:
|
||
passports = json.load(f)
|
||
|
||
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
|
||
catalog = json.load(f)
|
||
|
||
print(f"Initial: {len(ext)} ext symbols, {len(passports)} passports, {len(catalog)} catalog entries")
|
||
|
||
# ------------------------------------------------------------------------------
|
||
# 1. PARSE CLEAN CHUNAKOVA ARTICLES & PURGE BLOATED/CORRUPTED ONES
|
||
# ------------------------------------------------------------------------------
|
||
print(">>> Step 1: Parsing clean Chunakova articles...")
|
||
|
||
doc_ch = fitz.open(str(CHUNAKOVA_PDF))
|
||
|
||
def dehyphen_simple(text: str) -> str:
|
||
text = re.sub(r'([а-яёА-ЯЁA-Za-z]+)\xad\s*\n\s*([а-яёА-ЯЁA-Za-z]+)', r'\1\2', text)
|
||
text = re.sub(r'([а-яёА-ЯЁA-Za-z]+)\xad\s+([а-яёА-ЯЁA-Za-z]+)', r'\1\2', text)
|
||
text = text.replace('\xad', '')
|
||
return text
|
||
|
||
bad_prefixes = ('его ', 'ее ', 'их ', 'этот ', 'эта ', 'эти ', 'определение ', 'эпитеты ', 'символ ', 'как ', 'в ', 'по ', 'на ', 'при ', 'от ', 'с ', 'для ', 'из ', 'три ', 'четыре ', 'пять ', 'шесть ', 'семь ', 'восемь ', 'девять ', 'десять ')
|
||
|
||
clean_chunakova = {}
|
||
cur_h = None
|
||
cur_lines = []
|
||
|
||
for p in range(12, 285):
|
||
raw = doc_ch[p].get_text()
|
||
cleaned = dehyphen_simple(raw)
|
||
for line in cleaned.splitlines():
|
||
line = line.strip()
|
||
if not line or line.isdigit() or len(line) <= 2:
|
||
continue
|
||
m = re.match(r'^([А-ЯЁ][а-яёА-ЯЁ\s\-]{1,35}?)\s*[—–-]\s*(.*)', line)
|
||
if m:
|
||
head = m.group(1).strip()
|
||
rest = m.group(2).strip()
|
||
norm = head.lower()
|
||
if not any(norm.startswith(bp) for bp in bad_prefixes) and not any(c.isdigit() for c in head):
|
||
if cur_h and cur_lines:
|
||
txt = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
|
||
if len(txt) >= 25:
|
||
clean_chunakova[cur_h.lower()] = (cur_h, txt)
|
||
cur_h = head
|
||
cur_lines = [rest] if rest else []
|
||
continue
|
||
if cur_h:
|
||
cur_lines.append(line)
|
||
|
||
if cur_h and cur_lines:
|
||
txt = re.sub(r'\s+', ' ', ' '.join(cur_lines)).strip()
|
||
if len(txt) >= 25:
|
||
clean_chunakova[cur_h.lower()] = (cur_h, txt)
|
||
|
||
print(f"Extracted {len(clean_chunakova)} clean Chunakova articles in {time.time()-t0:.2f}s")
|
||
|
||
# Purge corrupted Chunakova entries from ext
|
||
chunakova_bloat = ['его символ', 'определение дахака', 'три сферы-стоянки', 'эпитеты ахримана']
|
||
for b_k in chunakova_bloat:
|
||
if b_k in ext:
|
||
del ext[b_k]
|
||
|
||
# Inject clean Chunakova articles
|
||
for norm_k, (term_c, text_c) in clean_chunakova.items():
|
||
if norm_k in ext:
|
||
# Update Chunakova source
|
||
sources = [s for s in ext[norm_k].get('sources', []) if 'Чунакова' not in s.get('author', '')]
|
||
sources.append({
|
||
'author': 'О.М. Чунакова',
|
||
'work': '«Пехлевийский словарь зороастрийских терминов»',
|
||
'excerpt': (text_c[:250] + '…') if len(text_c) > 250 else text_c,
|
||
'full_text': text_c
|
||
})
|
||
ext[norm_k]['sources'] = sources
|
||
else:
|
||
# Create concise entry if valid length
|
||
if len(text_c) >= 30:
|
||
ext[norm_k] = {
|
||
'term': term_c,
|
||
'normalized': norm_k,
|
||
'category_guess': 'Зороастрийская сакральная традиция',
|
||
'sources': [{
|
||
'author': 'О.М. Чунакова',
|
||
'work': '«Пехлевийский словарь зороастрийских терминов»',
|
||
'excerpt': (text_c[:250] + '…') if len(text_c) > 250 else text_c,
|
||
'full_text': text_c
|
||
}],
|
||
'illustrations': []
|
||
}
|
||
|
||
print(">>> Step 1 complete.")
|
||
|
||
# ------------------------------------------------------------------------------
|
||
# 2. PURGE GHOST EMPTY KEYS (97 KEYS) & MOVE VALUABLE ILLUSTRATIONS
|
||
# ------------------------------------------------------------------------------
|
||
print(">>> Step 2: Purging empty ghost keys and saving illustrations...")
|
||
|
||
# Move valuable illustrations to legitimate targets
|
||
ill_relocations = {
|
||
'огонь элемент': 'огонь',
|
||
'сердце символизируется лотосом': 'сердце',
|
||
'корзина с фруктами': 'фрукты',
|
||
'большие рыбы': 'рыба',
|
||
'фламандские пословицы': 'пословицы',
|
||
'меняла с женой': 'меняла',
|
||
'аллегории пяти чувств': 'пять чувств'
|
||
}
|
||
|
||
for ghost_k, target_k in ill_relocations.items():
|
||
if ghost_k in ext and target_k in ext:
|
||
for ill in ext[ghost_k].get('illustrations', []):
|
||
if not any(x.get('file') == ill.get('file') for x in ext[target_k].get('illustrations', [])):
|
||
ext[target_k].setdefault('illustrations', []).append(ill)
|
||
|
||
# Find and delete all items with 0 sources
|
||
ghosts_deleted = 0
|
||
for k in list(ext.keys()):
|
||
if not ext[k].get('sources'):
|
||
del ext[k]
|
||
ghosts_deleted += 1
|
||
|
||
print(f">>> Step 2 complete: Purged {ghosts_deleted} ghost keys with 0 sources.")
|
||
|
||
# ------------------------------------------------------------------------------
|
||
# 3. UNIVERSAL DE-HYPHENATION ENGINE & ARTIFACT REPAIR
|
||
# ------------------------------------------------------------------------------
|
||
print(">>> Step 3: Running universal de-hyphenation engine across all texts...")
|
||
|
||
compound_prefixes = {
|
||
'северо', 'юго', 'западно', 'восточно', 'темно', 'светло', 'ярко', 'бледно',
|
||
'военно', 'научно', 'социально', 'политико', 'физико', 'химико', 'историко',
|
||
'литературно', 'торгово', 'русско', 'греко', 'англо', 'франко', 'индо', 'китае',
|
||
'евро', 'афро', 'арабо', 'мусульмано', 'христиано', 'иудео', 'буддо', 'инь', 'ян'
|
||
}
|
||
|
||
particles = {'то', 'либо', 'нибудь', 'таки'}
|
||
|
||
def universal_clean_text(text: str) -> str:
|
||
if not text or not isinstance(text, str):
|
||
return text
|
||
|
||
# 1. Embedded footnotes / OCR captions inside words
|
||
text = re.sub(r'грече1\s+П\s+о\s+р\s+о\s+к[^\.\n]*?ского', 'греческого', text)
|
||
text = re.sub(r'по1\s+Г\s+о\s+р\s+г\s+у\s+л\s+ь\s+я[^\.\n]*?сланников', 'посланников', text)
|
||
text = re.sub(r'улы1\s+Вспомним[^\.\n]*?бающаяся', 'улыбающаяся', text)
|
||
text = re.sub(r'вер393\s+Сирена[^\.\n]*?сии', 'версии', text)
|
||
text = re.sub(r'отдыха141\s+Дерево[^\.\n]*?древо', 'отдыха', text)
|
||
text = re.sub(r'Восемь благоприятных символов\s+23\s+Будды', 'Восемь благоприятных символов Будды', text)
|
||
text = re.sub(r'1\s+С\s+у\s+з\s+н\s+и\s+ц\s+а\s*\.[^\.\n]*?э\s+л\s+е\s+м\s*', '', text)
|
||
|
||
# 2. Cut off trailing index catalog in 'ястреб-перепелятник'
|
||
if 'Птицы; Хаос; Ребенок; Колонны' in text:
|
||
text = text.split('Птицы; Хаос; Ребенок')[0].strip()
|
||
|
||
# 3. Soft hyphens (\xad)
|
||
text = re.sub(r'([а-яёА-ЯЁA-Za-z]{2,})\xad\s+([а-яёА-ЯЁA-Za-z]{2,})', r'\1\2', text)
|
||
text = text.replace('\xad', '')
|
||
|
||
# 4. Newline broken hyphens
|
||
def repl_nl(m):
|
||
w1, w2 = m.group(1), m.group(2)
|
||
if w1.lower() in compound_prefixes:
|
||
return f'{w1}-{w2}'
|
||
if w1.lower().endswith(('о', 'е')) and w2.lower() in ('и', 'или'):
|
||
return f'{w1}- {w2}'
|
||
return f'{w1}{w2}'
|
||
text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s*\n\s*([а-яё]{2,})', repl_nl, text)
|
||
|
||
# 5. In-line broken hyphenations: 'сло- во'
|
||
def repl_hyphen_space(m):
|
||
w1, w2 = m.group(1), m.group(2)
|
||
w1_low = w1.lower()
|
||
w2_low = w2.lower()
|
||
# Hanging hyphen: 'научно- и'
|
||
if w1_low.endswith(('о', 'е')) and w2_low in ('и', 'или'):
|
||
return f'{w1}- {w2}'
|
||
# Compound prefix: 'северо- восточный' -> 'северо-восточный'
|
||
if w1_low in compound_prefixes:
|
||
return f'{w1}-{w2}'
|
||
# Particles: 'какой- то' -> 'какой-то'
|
||
if w2_low in particles:
|
||
return f'{w1}-{w2}'
|
||
if w1_low in ('из', 'по') and w2_low in ('за', 'под', 'прежнему', 'новому', 'видимому'):
|
||
return f'{w1}-{w2}'
|
||
# Default: fuse word split
|
||
return f'{w1}{w2}'
|
||
|
||
# Double pass for chained hyphenations
|
||
text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)
|
||
text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)
|
||
|
||
# 6. Spacing before punctuation
|
||
text = re.sub(r'\s+([,.:;?!])', r'\1', text)
|
||
text = re.sub(r'\(\s+', '(', text)
|
||
text = re.sub(r'\s+\)', ')', text)
|
||
|
||
# 7. Normalize whitespace
|
||
text = re.sub(r'[ \t]{2,}', ' ', text)
|
||
return text.strip()
|
||
|
||
def clean_data_structure(data):
|
||
"""Recursively clean all strings in dicts/lists."""
|
||
if isinstance(data, dict):
|
||
return {k: clean_data_structure(v) for k, v in data.items()}
|
||
elif isinstance(data, list):
|
||
return [clean_data_structure(item) for item in data]
|
||
elif isinstance(data, str):
|
||
return universal_clean_text(data)
|
||
return data
|
||
|
||
ext = clean_data_structure(ext)
|
||
passports = clean_data_structure(passports)
|
||
catalog = clean_data_structure(catalog)
|
||
|
||
print(">>> Step 3 complete: All strings de-hyphenated and cleaned.")
|
||
|
||
# ------------------------------------------------------------------------------
|
||
# 4. FINAL SOURCE DEDUPLICATION
|
||
# ------------------------------------------------------------------------------
|
||
print(">>> Step 4: Final source deduplication...")
|
||
|
||
dedup_count = 0
|
||
for k, item in ext.items():
|
||
sources = item.get('sources', [])
|
||
if not sources:
|
||
continue
|
||
unique = []
|
||
seen = set()
|
||
for s in sources:
|
||
author = s.get('author', '').strip().lower()
|
||
work = s.get('work', '').strip().lower()
|
||
snippet = s.get('full_text', s.get('excerpt', ''))[:50].strip().lower()
|
||
sig = (author, work, snippet)
|
||
if sig not in seen:
|
||
seen.add(sig)
|
||
unique.append(s)
|
||
else:
|
||
dedup_count += 1
|
||
item['sources'] = unique
|
||
|
||
print(f">>> Step 4 complete: Deduplicated {dedup_count} sources.")
|
||
|
||
# ------------------------------------------------------------------------------
|
||
# 5. SAVE ALL DATABASES
|
||
# ------------------------------------------------------------------------------
|
||
print(">>> Step 5: Saving databases...")
|
||
|
||
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(ext, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(PASSPORTS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(STAGING_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||
|
||
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
||
|
||
print("=" * 80)
|
||
print(f"✅ DEEP CLEANING COMPLETE IN {time.time()-t0:.2f}s!")
|
||
print(f"Ext symbols: {len(ext)}")
|
||
print(f"Passports: {len(passports)}")
|
||
print(f"Catalog entries: {len(catalog)}")
|
||
print("=" * 80)
|