SillyTavern world artifacts, versioned in place
Tracked inside SillyTavern's live data directory rather than symlinked out of it. data/ is a bind mount into the container, so a symlink to a host path outside that mount resolves inside the container where the target does not exist -- broken to SillyTavern while looking correct on the host. The .gitignore is an allowlist, denying everything and re-including named paths, because this is live application data rather than a curated export. secrets.json is the reason: empty today because API keys go through LiteLLM with a placeholder, and it arms itself the moment a key is typed into the UI. This repo is public. A denylist protects only the paths someone remembered. Two of the four artifact types are not diffable as stored. Character cards are PNGs with the card JSON in a tEXt chunk, and personas live inside a 48 KB settings.json that SillyTavern rewrites on any change. st-export.py extracts both into _text/ so the reviewable artifact is text, one-way on purpose -- re-embedding JSON into a PNG is where a mistake corrupts authored content. Stock content is excluded: Seraphina's expression pack is 3.6 MiB of PNG the app regenerates on startup. The stock lorebook Eldoria.json stays -- 8 KB of readable JSON is a useful worked example of the format.
This commit is contained in:
Mikkeli
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
"""st-export — turn SillyTavern's un-diffable artifacts into reviewable text.
|
||||
|
||||
Two of the four things worth versioning here are not text:
|
||||
|
||||
character cards PNG images with the card JSON in a tEXt chunk
|
||||
personas buried inside settings.json, which is 48 KB of UI state
|
||||
that SillyTavern rewrites on any change
|
||||
|
||||
Committing the PNG gives you a binary blob; committing settings.json gives you a
|
||||
diff that churns every time a slider moves. Neither shows *"I changed her speech
|
||||
register"*, which is the only reason to version this at all.
|
||||
|
||||
So the PNG stays the artifact SillyTavern loads, and this writes the readable
|
||||
copy beside it:
|
||||
|
||||
_text/cards/<Name>.json extracted from the PNG's tEXt/zTXt chunk
|
||||
_text/personas.json extracted from settings.json
|
||||
|
||||
⚠ ONE-WAY, ON PURPOSE. This does not write back into SillyTavern. Editing a card
|
||||
by hand and re-embedding it into the PNG is a real operation, but it is one where
|
||||
a mistake corrupts authored content, so it is not something to do implicitly on a
|
||||
script run. Edit in the UI; run this; commit.
|
||||
|
||||
./st-export.py # regenerate _text/, then `git add -A`
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64, json, os, struct, sys, zlib
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
OUT = os.path.join(HERE, "_text")
|
||||
|
||||
|
||||
def png_text_chunks(path: str) -> dict[str, str]:
|
||||
"""Read tEXt and zTXt chunks with the stdlib — no Pillow dependency.
|
||||
|
||||
⚠ Character cards are the PNG spec being used as a container: v2 stores
|
||||
base64 JSON under the keyword `chara`, v3 under `ccv3`. The image itself is
|
||||
incidental.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
with open(path, "rb") as fh:
|
||||
if fh.read(8) != b"\x89PNG\r\n\x1a\n":
|
||||
return out
|
||||
while True:
|
||||
head = fh.read(8)
|
||||
if len(head) < 8:
|
||||
break
|
||||
length, ctype = struct.unpack(">I4s", head)
|
||||
data = fh.read(length)
|
||||
fh.read(4) # CRC, not verified — we only read
|
||||
if ctype == b"tEXt":
|
||||
k, _, v = data.partition(b"\x00")
|
||||
out[k.decode("latin-1")] = v.decode("latin-1")
|
||||
elif ctype == b"zTXt":
|
||||
k, _, rest = data.partition(b"\x00")
|
||||
if rest[:1] == b"\x00":
|
||||
try:
|
||||
out[k.decode("latin-1")] = zlib.decompress(rest[1:]).decode("utf-8")
|
||||
except Exception:
|
||||
pass
|
||||
elif ctype == b"IEND":
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def export_cards() -> int:
|
||||
src = os.path.join(HERE, "characters")
|
||||
dst = os.path.join(OUT, "cards")
|
||||
os.makedirs(dst, exist_ok=True)
|
||||
n = 0
|
||||
for name in sorted(os.listdir(src)) if os.path.isdir(src) else []:
|
||||
if not name.lower().endswith(".png"):
|
||||
continue
|
||||
# Stock demo cards are untracked (see .gitignore), so exporting them
|
||||
# would leave _text/cards/ describing files the repo does not hold.
|
||||
if name.startswith("default_"):
|
||||
continue
|
||||
chunks = png_text_chunks(os.path.join(src, name))
|
||||
raw = chunks.get("ccv3") or chunks.get("chara")
|
||||
if not raw:
|
||||
print(f" ⚠ {name}: no card data in the PNG — skipped")
|
||||
continue
|
||||
try:
|
||||
card = json.loads(base64.b64decode(raw).decode("utf-8"))
|
||||
except Exception as exc:
|
||||
print(f" ⚠ {name}: card data present but unreadable ({exc})")
|
||||
continue
|
||||
# sort_keys so a re-export of an unchanged card is a no-op diff
|
||||
with open(os.path.join(dst, name[:-4] + ".json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(card, fh, indent=2, ensure_ascii=False, sort_keys=True)
|
||||
fh.write("\n")
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def export_personas() -> int:
|
||||
p = os.path.join(HERE, "settings.json")
|
||||
if not os.path.exists(p):
|
||||
print(" ⚠ no settings.json — personas not exported")
|
||||
return 0
|
||||
d = json.load(open(p, encoding="utf-8"))
|
||||
pu = d.get("power_user") or {}
|
||||
# ⚠ Take ONLY the persona keys. settings.json also holds API endpoints and
|
||||
# every UI preference; copying it wholesale is how a public repo acquires
|
||||
# something it should not have.
|
||||
out = {k: pu.get(k) for k in
|
||||
("personas", "persona_descriptions", "default_persona", "persona_description")
|
||||
if pu.get(k) is not None}
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
with open(os.path.join(OUT, "personas.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(out, fh, indent=2, ensure_ascii=False, sort_keys=True)
|
||||
fh.write("\n")
|
||||
return len(out.get("personas") or {})
|
||||
|
||||
|
||||
def main() -> int:
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
cards = export_cards()
|
||||
personas = export_personas()
|
||||
worlds = len([f for f in os.listdir(os.path.join(HERE, "worlds"))
|
||||
if f.endswith(".json")]) if os.path.isdir(os.path.join(HERE, "worlds")) else 0
|
||||
print(f" _text/cards/ {cards} card(s) extracted from PNG")
|
||||
print(f" _text/personas.json {personas} persona(s)")
|
||||
print(f" worlds/ {worlds} lorebook(s) already JSON, tracked as-is")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user