feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets

scripts/test_comprehensive_clean.py

@@ -0,0 +1,107 @@
+import re
+import json
+
+compound_prefixes = {
+    'северо', 'юго', 'западно', 'восточно', 'темно', 'светло', 'ярко', 'бледно',
+    'военно', 'научно', 'социально', 'политико', 'физико', 'химико', 'историко',
+    'литературно', 'торгово', 'русско', 'греко', 'англо', 'франко', 'индо', 'китае',
+    'евро', 'афро', 'арабо', 'мусульмано', 'христиано', 'иудео', 'буддо', 'инь', 'ян'
+}
+
+suspension_words = {'и', 'или', 'а', 'да', 'либо', 'как'}
+particles = {'то', 'либо', 'нибудь', 'таки'}
+
+def clean_hyphens_and_artifacts(text: str) -> str:
+    if not text:
+        return ""
+    
+    # 1. Embedded footnotes / captions in words
+    text = re.sub(r'грече1\s+П\s+о\s+р\s+о\s+к[^\.\n]*?ского', 'греческого', text)
+    text = re.sub(r'по1\s+Г\s+о\s+р\s+г\s+у\s+л\s+ь\s+я[^\.\n]*?сланников', 'посланников', text)
+    text = re.sub(r'улы1\s+Вспомним[^\.\n]*?бающаяся', 'улыбающаяся', text)
+    text = re.sub(r'вер393\s+Сирена[^\.\n]*?сии', 'версии', text)
+    text = re.sub(r'отдыха141\s+Дерево[^\.\n]*?древо', 'отдыха', text)
+    text = re.sub(r'Восемь благоприятных символов\s+23\s+Будды', 'Восемь благоприятных символов Будды', text)
+    text = re.sub(r'1\s+С\s+у\s+з\s+н\s+и\s+ц\s+а\s*\.[^\.\n]*?э\s+л\s+е\s+м\s*', '', text)
+
+    # 2. Soft hyphens
+    text = re.sub(r'([а-яёА-ЯЁA-Za-z]{2,})\xad\s+([а-яёА-ЯЁA-Za-z]{2,})', r'\1\2', text)
+    text = text.replace('\xad', '')
+
+    # 3. Newline hyphens
+    def repl_nl(m):
+        w1, w2 = m.group(1), m.group(2)
+        if w1.lower() in compound_prefixes:
+            return f'{w1}-{w2}'
+        if w2.lower() in suspension_words:
+            return f'{w1}- {w2}'
+        return f'{w1}{w2}'
+    text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s*\n\s*([а-яё]{2,})', repl_nl, text)
+
+    # 4. In-line broken hyphenations: 'сло- во'
+    def repl_hyphen_space(m):
+        w1, w2 = m.group(1), m.group(2)
+        w1_low = w1.lower()
+        w2_low = w2.lower()
+        if w2_low in suspension_words:
+            return f'{w1}- {w2}'
+        if w1_low in compound_prefixes:
+            return f'{w1}-{w2}'
+        if w2_low in particles:
+            return f'{w1}-{w2}'
+        if w1_low in ('из', 'по') and w2_low in ('за', 'под', 'прежнему', 'новому', 'видимому'):
+            return f'{w1}-{w2}'
+        return f'{w1}{w2}'
+
+    # Run twice to catch chained splits e.g. 'мно- го- гранный'
+    text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)
+    text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)
+
+    # 5. Fix punctuation spacing
+    text = re.sub(r'\s+([,.:;?!])', r'\1', text)
+    text = re.sub(r'\(\s+', '(', text)
+    text = re.sub(r'\s+\)', ')', text)
+    
+    # 6. Normalize multiple spaces
+    text = re.sub(r'[ \t]{2,}', ' ', text)
+    return text.strip()
+
+with open('assets/data/oracle_external_symbols.json') as f:
+    ext = json.load(f)
+
+with open('assets/data/oracle_passports.json') as f:
+    passports = json.load(f)
+
+# Test before & after count of broken hyphens
+before_ext = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', s.get('full_text', ''))) for item in ext.values() for s in item.get('sources', []))
+before_pass = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', p.get('cultural_context', '') + ' ' + p.get('deep_psychology', '') + ' ' + p.get('short_meaning', ''))) for p in passports.values())
+
+print(f"Before: {before_ext} broken hyphens in ext, {before_pass} in passports")
+
+# Simulate cleaning
+cleaned_ext_count = 0
+for k, item in ext.items():
+    for s in item.get('sources', []):
+        old_txt = s.get('full_text', '')
+        new_txt = clean_hyphens_and_artifacts(old_txt)
+        if old_txt != new_txt:
+            cleaned_ext_count += 1
+            s['full_text'] = new_txt
+        if 'excerpt' in s:
+            s['excerpt'] = clean_hyphens_and_artifacts(s['excerpt'])
+
+cleaned_pass_count = 0
+for pid, p in passports.items():
+    for field in ['cultural_context', 'deep_psychology', 'short_meaning', 'archetype']:
+        if field in p and p[field]:
+            old_txt = p[field]
+            new_txt = clean_hyphens_and_artifacts(old_txt)
+            if old_txt != new_txt:
+                cleaned_pass_count += 1
+                p[field] = new_txt
+
+after_ext = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', s.get('full_text', ''))) for item in ext.values() for s in item.get('sources', []))
+after_pass = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', p.get('cultural_context', '') + ' ' + p.get('deep_psychology', '') + ' ' + p.get('short_meaning', ''))) for p in passports.values())
+
+print(f"After: {after_ext} broken hyphens in ext, {after_pass} in passports")
+print(f"Cleaned {cleaned_ext_count} source texts in ext, {cleaned_pass_count} fields in passports!")