sillytavern-world/st-export.py

#!/usr/bin/env python3
"""st-export — turn SillyTavern's un-diffable artifacts into reviewable text.

Two of the four things worth versioning here are not text:

    character cards   PNG images with the card JSON in a tEXt chunk
    personas          buried inside settings.json, which is 48 KB of UI state
                      that SillyTavern rewrites on any change

Committing the PNG gives you a binary blob; committing settings.json gives you a
diff that churns every time a slider moves. Neither shows *"I changed her speech
register"*, which is the only reason to version this at all.

So the PNG stays the artifact SillyTavern loads, and this writes the readable
copy beside it:

    _text/cards/<Name>.json      extracted from the PNG's tEXt/zTXt chunk
    _text/personas.json          extracted from settings.json

⚠ ONE-WAY, ON PURPOSE. This does not write back into SillyTavern. Editing a card
by hand and re-embedding it into the PNG is a real operation, but it is one where
a mistake corrupts authored content, so it is not something to do implicitly on a
script run. Edit in the UI; run this; commit.

    ./st-export.py            # regenerate _text/, then `git add -A`
"""
from __future__ import annotations

import base64, json, os, struct, sys, zlib

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "_text")


def png_text_chunks(path: str) -> dict[str, str]:
    """Read tEXt and zTXt chunks with the stdlib — no Pillow dependency.

    ⚠ Character cards are the PNG spec being used as a container: v2 stores
    base64 JSON under the keyword `chara`, v3 under `ccv3`. The image itself is
    incidental.
    """
    out: dict[str, str] = {}
    with open(path, "rb") as fh:
        if fh.read(8) != b"\x89PNG\r\n\x1a\n":
            return out
        while True:
            head = fh.read(8)
            if len(head) < 8:
                break
            length, ctype = struct.unpack(">I4s", head)
            data = fh.read(length)
            fh.read(4)                      # CRC, not verified — we only read
            if ctype == b"tEXt":
                k, _, v = data.partition(b"\x00")
                out[k.decode("latin-1")] = v.decode("latin-1")
            elif ctype == b"zTXt":
                k, _, rest = data.partition(b"\x00")
                if rest[:1] == b"\x00":
                    try:
                        out[k.decode("latin-1")] = zlib.decompress(rest[1:]).decode("utf-8")
                    except Exception:
                        pass
            elif ctype == b"IEND":
                break
    return out


def export_cards() -> int:
    src = os.path.join(HERE, "characters")
    dst = os.path.join(OUT, "cards")
    os.makedirs(dst, exist_ok=True)
    n = 0
    for name in sorted(os.listdir(src)) if os.path.isdir(src) else []:
        if not name.lower().endswith(".png"):
            continue
        # Stock demo cards are untracked (see .gitignore), so exporting them
        # would leave _text/cards/ describing files the repo does not hold.
        if name.startswith("default_"):
            continue
        chunks = png_text_chunks(os.path.join(src, name))
        raw = chunks.get("ccv3") or chunks.get("chara")
        if not raw:
            print(f"  ⚠ {name}: no card data in the PNG — skipped")
            continue
        try:
            card = json.loads(base64.b64decode(raw).decode("utf-8"))
        except Exception as exc:
            print(f"  ⚠ {name}: card data present but unreadable ({exc})")
            continue
        # sort_keys so a re-export of an unchanged card is a no-op diff
        with open(os.path.join(dst, name[:-4] + ".json"), "w", encoding="utf-8") as fh:
            json.dump(card, fh, indent=2, ensure_ascii=False, sort_keys=True)
            fh.write("\n")
        n += 1
    return n


def export_personas() -> int:
    p = os.path.join(HERE, "settings.json")
    if not os.path.exists(p):
        print("  ⚠ no settings.json — personas not exported")
        return 0
    d = json.load(open(p, encoding="utf-8"))
    pu = d.get("power_user") or {}
    # ⚠ Take ONLY the persona keys. settings.json also holds API endpoints and
    # every UI preference; copying it wholesale is how a public repo acquires
    # something it should not have.
    out = {k: pu.get(k) for k in
           ("personas", "persona_descriptions", "default_persona", "persona_description")
           if pu.get(k) is not None}
    os.makedirs(OUT, exist_ok=True)
    with open(os.path.join(OUT, "personas.json"), "w", encoding="utf-8") as fh:
        json.dump(out, fh, indent=2, ensure_ascii=False, sort_keys=True)
        fh.write("\n")
    return len(out.get("personas") or {})


def main() -> int:
    os.makedirs(OUT, exist_ok=True)
    cards = export_cards()
    personas = export_personas()
    worlds = len([f for f in os.listdir(os.path.join(HERE, "worlds"))
                  if f.endswith(".json")]) if os.path.isdir(os.path.join(HERE, "worlds")) else 0
    print(f"  _text/cards/     {cards} card(s) extracted from PNG")
    print(f"  _text/personas.json  {personas} persona(s)")
    print(f"  worlds/          {worlds} lorebook(s) already JSON, tracked as-is")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())