feat: complete academic symbol lexicon heal, 2-column Cooper extraction, and updated assets
This commit is contained in:
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
fix_illustrations_master.py
|
||||
===========================
|
||||
Глобальное исправление каталога и привязок иллюстраций:
|
||||
1. Восстановление префикса assets/illustrations/ для всех 916 файлов
|
||||
2. Полное удаление 28 фейковых виньеток Керлота (MD5 edb1a8f53f9b5502ae38e297711cdf6a)
|
||||
3. Полное удаление 146 страниц оглавления/введения Бира (soderzhanie, vvedenie)
|
||||
4. Исправление конкретных ложных привязок:
|
||||
- сверчок (убираем свастики)
|
||||
- свастика (убираем критского осьминога)
|
||||
- василиск (убираем вазу на могиле)
|
||||
- шип (убираем шлем Афины)
|
||||
- фиолетовый (убираем триграммы)
|
||||
- бамбук и баньян (убираем бабочку)
|
||||
- фаллос (убираем фейковую виньетку)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List
|
||||
|
||||
ROOT = Path("/home/alexives/Desktop/StudioProjects/999")
|
||||
CATALOG_PATH = ROOT / "assets/data/oracle_illustrations_catalog.json"
|
||||
PASSPORTS_PATH = ROOT / "assets/data/oracle_passports.json"
|
||||
STAGING_PATH = ROOT / "assets/data/oracle_passports_staging.json"
|
||||
EXTERNAL_PATH = ROOT / "assets/data/oracle_external_symbols.json"
|
||||
ILL_DIR = ROOT / "assets/illustrations"
|
||||
|
||||
FAKE_CIRLOT_MD5 = "edb1a8f53f9b5502ae38e297711cdf6a"
|
||||
|
||||
def normalize_path(p: str) -> str:
|
||||
if not p:
|
||||
return ""
|
||||
p = p.strip()
|
||||
if not p.startswith("assets/illustrations/"):
|
||||
fname = os.path.basename(p)
|
||||
return f"assets/illustrations/{fname}"
|
||||
return p
|
||||
|
||||
def main():
|
||||
print("=== НАЧАЛО ИСПРАВЛЕНИЯ ЦЕЛОСТНОСТИ ИЛЛЮСТРАЦИЙ ===")
|
||||
|
||||
with open(CATALOG_PATH, "r", encoding="utf-8") as f:
|
||||
catalog = json.load(f)
|
||||
|
||||
with open(PASSPORTS_PATH, "r", encoding="utf-8") as f:
|
||||
passports = json.load(f)
|
||||
|
||||
with open(STAGING_PATH, "r", encoding="utf-8") as f:
|
||||
staging = json.load(f)
|
||||
|
||||
with open(EXTERNAL_PATH, "r", encoding="utf-8") as f:
|
||||
external = json.load(f)
|
||||
|
||||
# 1. Поиск запрещенных / фейковых файлов
|
||||
blacklisted_files = set()
|
||||
for f in ILL_DIR.glob("*"):
|
||||
if f.is_file():
|
||||
fname = f.name
|
||||
# Оглавление и введение Бира
|
||||
if "soderzhanie" in fname.lower() or "vvedenie" in fname.lower():
|
||||
blacklisted_files.add(fname)
|
||||
blacklisted_files.add(f"assets/illustrations/{fname}")
|
||||
# Фейковая виньетка Керлота
|
||||
try:
|
||||
h = hashlib.md5(f.read_bytes()).hexdigest()
|
||||
if h == FAKE_CIRLOT_MD5:
|
||||
blacklisted_files.add(fname)
|
||||
blacklisted_files.add(f"assets/illustrations/{fname}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f"Обнаружено {len(blacklisted_files)} запрещенных файлов (виньетки и оглавления).")
|
||||
|
||||
# Удаляем запрещенные файлы с диска
|
||||
deleted_disk_count = 0
|
||||
for bf in blacklisted_files:
|
||||
p = ROOT / bf
|
||||
if p.exists():
|
||||
try:
|
||||
p.unlink()
|
||||
deleted_disk_count += 1
|
||||
except Exception:
|
||||
pass
|
||||
print(f"Удалено с диска: {deleted_disk_count} файлов.")
|
||||
|
||||
# 2. Фильтрация и исправление путей в Catalog
|
||||
new_catalog = []
|
||||
for item in catalog:
|
||||
raw_f = item.get("file") or item.get("path") or ""
|
||||
fname = os.path.basename(raw_f)
|
||||
if fname in blacklisted_files or raw_f in blacklisted_files:
|
||||
continue
|
||||
|
||||
fixed_path = normalize_path(raw_f)
|
||||
item["file"] = fixed_path
|
||||
if "path" in item:
|
||||
item["path"] = fixed_path
|
||||
|
||||
# Проверяем конкретные ложные привязки
|
||||
norm = item.get("normalized", "").lower()
|
||||
cap = (item.get("caption") or "").lower()
|
||||
|
||||
# сверчок со значком свастики из Уильямса
|
||||
if norm == "сверчок" and "wil_370" in fixed_path:
|
||||
continue
|
||||
|
||||
# осьминог у свастики
|
||||
if norm == "свастика" and ("осьминог" in cap or "cooper_p229" in fixed_path):
|
||||
continue
|
||||
|
||||
# ваза у василиска
|
||||
if norm == "василиск" and ("ваза" in cap or "cooper_p30" in fixed_path):
|
||||
continue
|
||||
|
||||
# шлем у шипа
|
||||
if norm == "шип" and ("шлем" in cap or "cooper_p385" in fixed_path):
|
||||
continue
|
||||
|
||||
# триграммы у фиолетового
|
||||
if norm == "фиолетовый" and "cooper_p357" in fixed_path:
|
||||
continue
|
||||
|
||||
# бабочка у бамбука и баньяна
|
||||
if norm in ("бамбук", "баньян") and "cooper_p17" in fixed_path:
|
||||
continue
|
||||
|
||||
# Проверяем наличие файла на диске
|
||||
if (ROOT / fixed_path).exists():
|
||||
new_catalog.append(item)
|
||||
|
||||
print(f"Каталог очищен: осталось {len(new_catalog)} валидных иллюстраций (было {len(catalog)}).")
|
||||
|
||||
# 3. Фильтрация и нормализация в Passports
|
||||
for db_name, db in [("passports", passports), ("staging", staging)]:
|
||||
for pid, p in db.items():
|
||||
old_ills = p.get("illustrations", [])
|
||||
new_ills = []
|
||||
for ill in old_ills:
|
||||
raw_f = ill.get("file") or ill.get("path") or ""
|
||||
fname = os.path.basename(raw_f)
|
||||
if fname in blacklisted_files or raw_f in blacklisted_files:
|
||||
continue
|
||||
|
||||
fixed_path = normalize_path(raw_f)
|
||||
ill["file"] = fixed_path
|
||||
if "path" in ill:
|
||||
ill["path"] = fixed_path
|
||||
|
||||
w = p.get("word", "").lower()
|
||||
cap = (ill.get("caption") or "").lower()
|
||||
|
||||
if w == "сверчок" and "wil_370" in fixed_path:
|
||||
continue
|
||||
if w == "свастика" and ("осьминог" in cap or "cooper_p229" in fixed_path):
|
||||
continue
|
||||
if w == "василиск" and ("ваза" in cap or "cooper_p30" in fixed_path):
|
||||
continue
|
||||
if w == "шип" and ("шлем" in cap or "cooper_p385" in fixed_path):
|
||||
continue
|
||||
if w == "фиолетовый" and "cooper_p357" in fixed_path:
|
||||
continue
|
||||
if w in ("бамбук", "баньян") and "cooper_p17" in fixed_path:
|
||||
continue
|
||||
|
||||
if (ROOT / fixed_path).exists():
|
||||
new_ills.append(ill)
|
||||
|
||||
p["illustrations"] = new_ills
|
||||
|
||||
# 4. Фильтрация и нормализация в External Symbols
|
||||
for term, obj in external.items():
|
||||
old_ills = obj.get("illustrations", [])
|
||||
new_ills = []
|
||||
for ill in old_ills:
|
||||
raw_f = ill.get("file") or ill.get("path") or ""
|
||||
fname = os.path.basename(raw_f)
|
||||
if fname in blacklisted_files or raw_f in blacklisted_files:
|
||||
continue
|
||||
|
||||
fixed_path = normalize_path(raw_f)
|
||||
ill["file"] = fixed_path
|
||||
if "path" in ill:
|
||||
ill["path"] = fixed_path
|
||||
|
||||
t_lower = term.lower()
|
||||
cap = (ill.get("caption") or "").lower()
|
||||
|
||||
if t_lower == "сверчок" and "wil_370" in fixed_path:
|
||||
continue
|
||||
if t_lower == "свастика" and ("осьминог" in cap or "cooper_p229" in fixed_path):
|
||||
continue
|
||||
if t_lower == "василиск" and ("ваза" in cap or "cooper_p30" in fixed_path):
|
||||
continue
|
||||
if t_lower == "шип" and ("шлем" in cap or "cooper_p385" in fixed_path):
|
||||
continue
|
||||
if t_lower == "фиолетовый" and "cooper_p357" in fixed_path:
|
||||
continue
|
||||
if t_lower in ("бамбук", "баньян") and "cooper_p17" in fixed_path:
|
||||
continue
|
||||
|
||||
if (ROOT / fixed_path).exists():
|
||||
new_ills.append(ill)
|
||||
|
||||
obj["illustrations"] = new_ills
|
||||
|
||||
# 5. Сохранение всех JSON баз
|
||||
with open(CATALOG_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(new_catalog, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(PASSPORTS_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(passports, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(STAGING_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(staging, f, ensure_ascii=False, indent=2)
|
||||
|
||||
with open(EXTERNAL_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(external, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print("Все JSON базы данных успешно обновлены и сохранены!")
|
||||
print("=== ЗАВЕРШЕНО ===")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user