60 lines
2.5 KiB
Python
60 lines
2.5 KiB
Python
import json
|
||
import re
|
||
from pathlib import Path
|
||
|
||
BASE_DIR = Path('/home/alexives/Desktop/StudioProjects/999')
|
||
EXT_SYMBOLS_PATH = BASE_DIR / 'assets/data/oracle_external_symbols.json'
|
||
PASSPORTS_PATH = BASE_DIR / 'assets/data/oracle_passports.json'
|
||
STAGING_PATH = BASE_DIR / 'assets/data/oracle_passports_staging.json'
|
||
CATALOG_PATH = BASE_DIR / 'assets/data/oracle_illustrations_catalog.json'
|
||
|
||
with open(EXT_SYMBOLS_PATH, 'r', encoding='utf-8') as f:
|
||
ext = json.load(f)
|
||
|
||
bad_keys = []
|
||
for k in ext.keys():
|
||
# Check digits
|
||
if any(c.isdigit() for c in k):
|
||
bad_keys.append(k)
|
||
continue
|
||
# Check garbage phrases
|
||
if any(phrase in k for phrase in [
|
||
'дерево 7', 'см.', 'рис.', 'богатую символику', 'после обряда', 'вое',
|
||
'на картине изображено', 'живописные', 'грецкий орех в данном',
|
||
'жук-олень может в данном', 'в алхимии', 'в магии', 'в исламе',
|
||
'у китайцев', 'у евреев', 'у христиан', 'у греков', 'у кельтов',
|
||
'у маори', 'у бушменов', 'то же в', 'числа 3', 'числа 2', 'числа 1',
|
||
'содержание дерево', 'предисловие к русскому'
|
||
]):
|
||
bad_keys.append(k)
|
||
continue
|
||
# Check sentence fragments (>= 4 words that are not compound symbol names)
|
||
words = k.split()
|
||
if len(words) >= 4 and not any(k.startswith(p) for p in ['семь ', 'четыре ', 'пять ', 'восемь ', 'двенадцать ']):
|
||
bad_keys.append(k)
|
||
continue
|
||
|
||
print(f"Total bad keys identified for removal: {len(bad_keys)}")
|
||
for bk in bad_keys[:15]:
|
||
print(" -", repr(bk))
|
||
|
||
# Remove bad keys
|
||
for bk in bad_keys:
|
||
del ext[bk]
|
||
|
||
# Also remove Cooper / Kortunova / Timofey from sources in remaining keys so we can re-add them cleanly
|
||
for k, v in ext.items():
|
||
v['sources'] = [s for s in v.get('sources', []) if s.get('author') not in ['Дж. Купер', 'Н.Д. Кортунова', 'Тимофей из Газы']]
|
||
|
||
with open(EXT_SYMBOLS_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(ext, f, ensure_ascii=False, indent=2)
|
||
|
||
# Update catalog
|
||
with open(CATALOG_PATH, 'r', encoding='utf-8') as f:
|
||
cat = json.load(f)
|
||
cat = [c for c in cat if c.get('normalized') not in set(bad_keys)]
|
||
with open(CATALOG_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(cat, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"Purge complete! Remaining clean external symbols: {len(ext)}")
|