108 lines
5.2 KiB
Python
108 lines
5.2 KiB
Python
import re
|
||
import json
|
||
|
||
compound_prefixes = {
|
||
'северо', 'юго', 'западно', 'восточно', 'темно', 'светло', 'ярко', 'бледно',
|
||
'военно', 'научно', 'социально', 'политико', 'физико', 'химико', 'историко',
|
||
'литературно', 'торгово', 'русско', 'греко', 'англо', 'франко', 'индо', 'китае',
|
||
'евро', 'афро', 'арабо', 'мусульмано', 'христиано', 'иудео', 'буддо', 'инь', 'ян'
|
||
}
|
||
|
||
suspension_words = {'и', 'или', 'а', 'да', 'либо', 'как'}
|
||
particles = {'то', 'либо', 'нибудь', 'таки'}
|
||
|
||
def clean_hyphens_and_artifacts(text: str) -> str:
|
||
if not text:
|
||
return ""
|
||
|
||
# 1. Embedded footnotes / captions in words
|
||
text = re.sub(r'грече1\s+П\s+о\s+р\s+о\s+к[^\.\n]*?ского', 'греческого', text)
|
||
text = re.sub(r'по1\s+Г\s+о\s+р\s+г\s+у\s+л\s+ь\s+я[^\.\n]*?сланников', 'посланников', text)
|
||
text = re.sub(r'улы1\s+Вспомним[^\.\n]*?бающаяся', 'улыбающаяся', text)
|
||
text = re.sub(r'вер393\s+Сирена[^\.\n]*?сии', 'версии', text)
|
||
text = re.sub(r'отдыха141\s+Дерево[^\.\n]*?древо', 'отдыха', text)
|
||
text = re.sub(r'Восемь благоприятных символов\s+23\s+Будды', 'Восемь благоприятных символов Будды', text)
|
||
text = re.sub(r'1\s+С\s+у\s+з\s+н\s+и\s+ц\s+а\s*\.[^\.\n]*?э\s+л\s+е\s+м\s*', '', text)
|
||
|
||
# 2. Soft hyphens
|
||
text = re.sub(r'([а-яёА-ЯЁA-Za-z]{2,})\xad\s+([а-яёА-ЯЁA-Za-z]{2,})', r'\1\2', text)
|
||
text = text.replace('\xad', '')
|
||
|
||
# 3. Newline hyphens
|
||
def repl_nl(m):
|
||
w1, w2 = m.group(1), m.group(2)
|
||
if w1.lower() in compound_prefixes:
|
||
return f'{w1}-{w2}'
|
||
if w2.lower() in suspension_words:
|
||
return f'{w1}- {w2}'
|
||
return f'{w1}{w2}'
|
||
text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s*\n\s*([а-яё]{2,})', repl_nl, text)
|
||
|
||
# 4. In-line broken hyphenations: 'сло- во'
|
||
def repl_hyphen_space(m):
|
||
w1, w2 = m.group(1), m.group(2)
|
||
w1_low = w1.lower()
|
||
w2_low = w2.lower()
|
||
if w2_low in suspension_words:
|
||
return f'{w1}- {w2}'
|
||
if w1_low in compound_prefixes:
|
||
return f'{w1}-{w2}'
|
||
if w2_low in particles:
|
||
return f'{w1}-{w2}'
|
||
if w1_low in ('из', 'по') and w2_low in ('за', 'под', 'прежнему', 'новому', 'видимому'):
|
||
return f'{w1}-{w2}'
|
||
return f'{w1}{w2}'
|
||
|
||
# Run twice to catch chained splits e.g. 'мно- го- гранный'
|
||
text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)
|
||
text = re.sub(r'([а-яёА-ЯЁ]{2,})[-\u2010\u2011]\s+([а-яё]{2,})', repl_hyphen_space, text)
|
||
|
||
# 5. Fix punctuation spacing
|
||
text = re.sub(r'\s+([,.:;?!])', r'\1', text)
|
||
text = re.sub(r'\(\s+', '(', text)
|
||
text = re.sub(r'\s+\)', ')', text)
|
||
|
||
# 6. Normalize multiple spaces
|
||
text = re.sub(r'[ \t]{2,}', ' ', text)
|
||
return text.strip()
|
||
|
||
with open('assets/data/oracle_external_symbols.json') as f:
|
||
ext = json.load(f)
|
||
|
||
with open('assets/data/oracle_passports.json') as f:
|
||
passports = json.load(f)
|
||
|
||
# Test before & after count of broken hyphens
|
||
before_ext = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', s.get('full_text', ''))) for item in ext.values() for s in item.get('sources', []))
|
||
before_pass = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', p.get('cultural_context', '') + ' ' + p.get('deep_psychology', '') + ' ' + p.get('short_meaning', ''))) for p in passports.values())
|
||
|
||
print(f"Before: {before_ext} broken hyphens in ext, {before_pass} in passports")
|
||
|
||
# Simulate cleaning
|
||
cleaned_ext_count = 0
|
||
for k, item in ext.items():
|
||
for s in item.get('sources', []):
|
||
old_txt = s.get('full_text', '')
|
||
new_txt = clean_hyphens_and_artifacts(old_txt)
|
||
if old_txt != new_txt:
|
||
cleaned_ext_count += 1
|
||
s['full_text'] = new_txt
|
||
if 'excerpt' in s:
|
||
s['excerpt'] = clean_hyphens_and_artifacts(s['excerpt'])
|
||
|
||
cleaned_pass_count = 0
|
||
for pid, p in passports.items():
|
||
for field in ['cultural_context', 'deep_psychology', 'short_meaning', 'archetype']:
|
||
if field in p and p[field]:
|
||
old_txt = p[field]
|
||
new_txt = clean_hyphens_and_artifacts(old_txt)
|
||
if old_txt != new_txt:
|
||
cleaned_pass_count += 1
|
||
p[field] = new_txt
|
||
|
||
after_ext = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', s.get('full_text', ''))) for item in ext.values() for s in item.get('sources', []))
|
||
after_pass = sum(len(re.findall(r'([а-яёА-ЯЁ]{2,})[-\xad\u2010\u2011]\s+([а-яё]{2,})', p.get('cultural_context', '') + ' ' + p.get('deep_psychology', '') + ' ' + p.get('short_meaning', ''))) for p in passports.values())
|
||
|
||
print(f"After: {after_ext} broken hyphens in ext, {after_pass} in passports")
|
||
print(f"Cleaned {cleaned_ext_count} source texts in ext, {cleaned_pass_count} fields in passports!")
|