mikkeli
4d3bf61d56
_text/ is derived from the PNG tEXt chunks and settings.json, and derived data regenerated by hand goes stale silently -- which is worse than absent, because a stale card reads as current. A pre-commit hook now regenerates and stages it. Blocking, unlike the halogen repo's advisory doc-check: that one reports a judgement call, this one rebuilds a file. Install with `./st-export.py --install-hook`, since hooks are not versioned. Verified by renaming a persona in the untracked settings.json and committing WITHOUT running the export -- _text/personas.json updated itself and was included. Test edit reverted.
165 lines
6.4 KiB
Python
Executable File
165 lines
6.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""st-export — turn SillyTavern's un-diffable artifacts into reviewable text.
|
|
|
|
Two of the four things worth versioning here are not text:
|
|
|
|
character cards PNG images with the card JSON in a tEXt chunk
|
|
personas buried inside settings.json, which is 48 KB of UI state
|
|
that SillyTavern rewrites on any change
|
|
|
|
Committing the PNG gives you a binary blob; committing settings.json gives you a
|
|
diff that churns every time a slider moves. Neither shows *"I changed her speech
|
|
register"*, which is the only reason to version this at all.
|
|
|
|
So the PNG stays the artifact SillyTavern loads, and this writes the readable
|
|
copy beside it:
|
|
|
|
_text/cards/<Name>.json extracted from the PNG's tEXt/zTXt chunk
|
|
_text/personas.json extracted from settings.json
|
|
|
|
⚠ ONE-WAY, ON PURPOSE. This does not write back into SillyTavern. Editing a card
|
|
by hand and re-embedding it into the PNG is a real operation, but it is one where
|
|
a mistake corrupts authored content, so it is not something to do implicitly on a
|
|
script run. Edit in the UI; run this; commit.
|
|
|
|
./st-export.py # regenerate _text/, then `git add -A`
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64, json, os, struct, sys, zlib
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
OUT = os.path.join(HERE, "_text")
|
|
|
|
|
|
def png_text_chunks(path: str) -> dict[str, str]:
|
|
"""Read tEXt and zTXt chunks with the stdlib — no Pillow dependency.
|
|
|
|
⚠ Character cards are the PNG spec being used as a container: v2 stores
|
|
base64 JSON under the keyword `chara`, v3 under `ccv3`. The image itself is
|
|
incidental.
|
|
"""
|
|
out: dict[str, str] = {}
|
|
with open(path, "rb") as fh:
|
|
if fh.read(8) != b"\x89PNG\r\n\x1a\n":
|
|
return out
|
|
while True:
|
|
head = fh.read(8)
|
|
if len(head) < 8:
|
|
break
|
|
length, ctype = struct.unpack(">I4s", head)
|
|
data = fh.read(length)
|
|
fh.read(4) # CRC, not verified — we only read
|
|
if ctype == b"tEXt":
|
|
k, _, v = data.partition(b"\x00")
|
|
out[k.decode("latin-1")] = v.decode("latin-1")
|
|
elif ctype == b"zTXt":
|
|
k, _, rest = data.partition(b"\x00")
|
|
if rest[:1] == b"\x00":
|
|
try:
|
|
out[k.decode("latin-1")] = zlib.decompress(rest[1:]).decode("utf-8")
|
|
except Exception:
|
|
pass
|
|
elif ctype == b"IEND":
|
|
break
|
|
return out
|
|
|
|
|
|
def export_cards() -> int:
|
|
src = os.path.join(HERE, "characters")
|
|
dst = os.path.join(OUT, "cards")
|
|
os.makedirs(dst, exist_ok=True)
|
|
n = 0
|
|
for name in sorted(os.listdir(src)) if os.path.isdir(src) else []:
|
|
if not name.lower().endswith(".png"):
|
|
continue
|
|
# Stock demo cards are untracked (see .gitignore), so exporting them
|
|
# would leave _text/cards/ describing files the repo does not hold.
|
|
if name.startswith("default_"):
|
|
continue
|
|
chunks = png_text_chunks(os.path.join(src, name))
|
|
raw = chunks.get("ccv3") or chunks.get("chara")
|
|
if not raw:
|
|
print(f" ⚠ {name}: no card data in the PNG — skipped")
|
|
continue
|
|
try:
|
|
card = json.loads(base64.b64decode(raw).decode("utf-8"))
|
|
except Exception as exc:
|
|
print(f" ⚠ {name}: card data present but unreadable ({exc})")
|
|
continue
|
|
# sort_keys so a re-export of an unchanged card is a no-op diff
|
|
with open(os.path.join(dst, name[:-4] + ".json"), "w", encoding="utf-8") as fh:
|
|
json.dump(card, fh, indent=2, ensure_ascii=False, sort_keys=True)
|
|
fh.write("\n")
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def export_personas() -> int:
|
|
p = os.path.join(HERE, "settings.json")
|
|
if not os.path.exists(p):
|
|
print(" ⚠ no settings.json — personas not exported")
|
|
return 0
|
|
d = json.load(open(p, encoding="utf-8"))
|
|
pu = d.get("power_user") or {}
|
|
# ⚠ Take ONLY the persona keys. settings.json also holds API endpoints and
|
|
# every UI preference; copying it wholesale is how a public repo acquires
|
|
# something it should not have.
|
|
out = {k: pu.get(k) for k in
|
|
("personas", "persona_descriptions", "default_persona", "persona_description")
|
|
if pu.get(k) is not None}
|
|
os.makedirs(OUT, exist_ok=True)
|
|
with open(os.path.join(OUT, "personas.json"), "w", encoding="utf-8") as fh:
|
|
json.dump(out, fh, indent=2, ensure_ascii=False, sort_keys=True)
|
|
fh.write("\n")
|
|
return len(out.get("personas") or {})
|
|
|
|
|
|
HOOK = """#!/usr/bin/env bash
|
|
# Installed by st-export.py --install-hook.
|
|
# Regenerates _text/ from the PNGs and settings.json, then stages it, so the
|
|
# readable copy can never drift from the artifact SillyTavern actually loads.
|
|
set -e
|
|
cd "$(git rev-parse --show-toplevel)"
|
|
python3 ./st-export.py >/dev/null
|
|
git add -A _text
|
|
"""
|
|
|
|
|
|
def install_hook() -> int:
|
|
import subprocess
|
|
root = subprocess.run(["git", "rev-parse", "--git-dir"], capture_output=True,
|
|
text=True).stdout.strip()
|
|
if not root:
|
|
print("not a git repository")
|
|
return 1
|
|
path = os.path.join(root, "hooks", "pre-commit")
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
open(path, "w").write(HOOK)
|
|
os.chmod(path, 0o755)
|
|
print(f"armed: {path}")
|
|
# ⚠ BLOCKING ON PURPOSE, unlike the halogen repo's advisory doc-check. That
|
|
# one reports a judgement call; this one regenerates a derived file. If it
|
|
# fails, _text/ would silently describe a card that no longer exists — and a
|
|
# stale derived copy is worse than no copy, because it reads as current.
|
|
print("⚠ runs on every commit; a failure blocks it, which is the point")
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
if "--install-hook" in sys.argv:
|
|
return install_hook()
|
|
os.makedirs(OUT, exist_ok=True)
|
|
cards = export_cards()
|
|
personas = export_personas()
|
|
worlds = len([f for f in os.listdir(os.path.join(HERE, "worlds"))
|
|
if f.endswith(".json")]) if os.path.isdir(os.path.join(HERE, "worlds")) else 0
|
|
print(f" _text/cards/ {cards} card(s) extracted from PNG")
|
|
print(f" _text/personas.json {personas} persona(s)")
|
|
print(f" worlds/ {worlds} lorebook(s) already JSON, tracked as-is")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|