feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets

2026-10-09 15:26:58 +05:00
parent 7f91366201
commit 0b12e4583b
244 changed files with 611461 additions and 1617 deletions
@@ -0,0 +1,59 @@
+import json
+import re
+from pathlib import Path
+
+BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
+EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
+PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
+STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
+CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
+
+with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
+    ext = json.load(f)
+
+bad_keys = []
+for k in ext.keys():
+    # Check digits
+    if any(c.isdigit() for c in k):
+        bad_keys.append(k)
+        continue
+    # Check garbage phrases
+    if any(phrase in k for phrase in [
+        'дерево 7', 'см.', 'рис.', 'богатую символику', 'после обряда', 'вое',
+        'на картине изображено', 'живописные', 'грецкий орех в данном',
+        'жук-олень может в данном', 'в алхимии', 'в магии', 'в исламе',
+        'у китайцев', 'у евреев', 'у христиан', 'у греков', 'у кельтов',
+        'у маори', 'у бушменов', 'то же в', 'числа 3', 'числа 2', 'числа 1',
+        'содержание дерево', 'предисловие к русскому'
+    ]):
+        bad_keys.append(k)
+        continue
+    # Check sentence fragments (>= 4 words that are not compound symbol names)
+    words = k.split()
+    if len(words) >= 4 and not any(k.startswith(p) for p in ['семь ', 'четыре ', 'пять ', 'восемь ', 'двенадцать ']):
+        bad_keys.append(k)
+        continue
+
+print(f"Total bad keys identified for removal: {len(bad_keys)}")
+for bk in bad_keys[:15]:
+    print("  -", repr(bk))
+
+# Remove bad keys
+for bk in bad_keys:
+    del ext[bk]
+
+# Also remove Cooper / Kortunova / Timofey from sources in remaining keys so we can re-add them cleanly
+for k, v in ext.items():
+    v['sources'] = [s for s in v.get('sources', []) if s.get('author') not in ['Дж. Купер', 'Н.Д. Кортунова', 'Тимофей из Газы']]
+
+with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
+    json.dump(ext, f, ensure_ascii=False, indent=2)
+
+# Update catalog
+with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
+    cat = json.load(f)
+cat = [c for c in cat if c.get('normalized') not in set(bad_keys)]
+with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
+    json.dump(cat, f, ensure_ascii=False, indent=2)
+
+print(f"Purge complete! Remaining clean external symbols: {len(ext)}")