sillytavern-world/st-export.py

#!/usr/bin/env python3
"""st-export — turn SillyTavern's un-diffable artifacts into reviewable text.

Two of the four things worth versioning here are not text:

    character cards   PNG images with the card JSON in a tEXt chunk
    personas          buried inside settings.json, which is 48 KB of UI state
                      that SillyTavern rewrites on any change

Committing the PNG gives you a binary blob; committing settings.json gives you a
diff that churns every time a slider moves. Neither shows *"I changed her speech
register"*, which is the only reason to version this at all.

So the PNG stays the artifact SillyTavern loads, and this writes the readable
copy beside it:

    _text/cards/<Name>.json      extracted from the PNG's tEXt/zTXt chunk
    _text/personas.json          extracted from settings.json

⚠ ONE-WAY, ON PURPOSE. This does not write back into SillyTavern. Editing a card
by hand and re-embedding it into the PNG is a real operation, but it is one where
a mistake corrupts authored content, so it is not something to do implicitly on a
script run. Edit in the UI; run this; commit.

    ./st-export.py            # regenerate _text/, then `git add -A`
"""
from __future__ import annotations

import base64, json, os, struct, sys, zlib

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "_text")


def png_text_chunks(path: str) -> dict[str, str]:
    """Read tEXt and zTXt chunks with the stdlib — no Pillow dependency.

    ⚠ Character cards are the PNG spec being used as a container: v2 stores
    base64 JSON under the keyword `chara`, v3 under `ccv3`. The image itself is
    incidental.
    """
    out: dict[str, str] = {}
    with open(path, "rb") as fh:
        if fh.read(8) != b"\x89PNG\r\n\x1a\n":
            return out
        while True:
            head = fh.read(8)
            if len(head) < 8:
                break
            length, ctype = struct.unpack(">I4s", head)
            data = fh.read(length)
            fh.read(4)                      # CRC, not verified — we only read
            if ctype == b"tEXt":
                k, _, v = data.partition(b"\x00")
                out[k.decode("latin-1")] = v.decode("latin-1")
            elif ctype == b"zTXt":
                k, _, rest = data.partition(b"\x00")
                if rest[:1] == b"\x00":
                    try:
                        out[k.decode("latin-1")] = zlib.decompress(rest[1:]).decode("utf-8")
                    except Exception:
                        pass
            elif ctype == b"IEND":
                break
    return out


def export_cards() -> int:
    src = os.path.join(HERE, "characters")
    dst = os.path.join(OUT, "cards")
    os.makedirs(dst, exist_ok=True)
    n = 0
    for name in sorted(os.listdir(src)) if os.path.isdir(src) else []:
        if not name.lower().endswith(".png"):
            continue
        # Stock demo cards are untracked (see .gitignore), so exporting them
        # would leave _text/cards/ describing files the repo does not hold.
        if name.startswith("default_"):
            continue
        chunks = png_text_chunks(os.path.join(src, name))
        raw = chunks.get("ccv3") or chunks.get("chara")
        if not raw:
            print(f"  ⚠ {name}: no card data in the PNG — skipped")
            continue
        try:
            card = json.loads(base64.b64decode(raw).decode("utf-8"))
        except Exception as exc:
            print(f"  ⚠ {name}: card data present but unreadable ({exc})")
            continue
        # sort_keys so a re-export of an unchanged card is a no-op diff
        with open(os.path.join(dst, name[:-4] + ".json"), "w", encoding="utf-8") as fh:
            json.dump(card, fh, indent=2, ensure_ascii=False, sort_keys=True)
            fh.write("\n")
        n += 1
    return n


def export_personas() -> int:
    p = os.path.join(HERE, "settings.json")
    if not os.path.exists(p):
        print("  ⚠ no settings.json — personas not exported")
        return 0
    d = json.load(open(p, encoding="utf-8"))
    pu = d.get("power_user") or {}
    # ⚠ Take ONLY the persona keys. settings.json also holds API endpoints and
    # every UI preference; copying it wholesale is how a public repo acquires
    # something it should not have.
    out = {k: pu.get(k) for k in
           ("personas", "persona_descriptions", "default_persona", "persona_description")
           if pu.get(k) is not None}
    os.makedirs(OUT, exist_ok=True)
    with open(os.path.join(OUT, "personas.json"), "w", encoding="utf-8") as fh:
        json.dump(out, fh, indent=2, ensure_ascii=False, sort_keys=True)
        fh.write("\n")
    return len(out.get("personas") or {})


HOOK = """#!/usr/bin/env bash
# Installed by st-export.py --install-hook.
# Regenerates _text/ from the PNGs and settings.json, then stages it, so the
# readable copy can never drift from the artifact SillyTavern actually loads.
set -e
cd "$(git rev-parse --show-toplevel)"
python3 ./st-export.py >/dev/null
git add -A _text
"""


def install_hook() -> int:
    import subprocess
    root = subprocess.run(["git", "rev-parse", "--git-dir"], capture_output=True,
                          text=True).stdout.strip()
    if not root:
        print("not a git repository")
        return 1
    path = os.path.join(root, "hooks", "pre-commit")
    os.makedirs(os.path.dirname(path), exist_ok=True)
    open(path, "w").write(HOOK)
    os.chmod(path, 0o755)
    print(f"armed: {path}")
    # ⚠ BLOCKING ON PURPOSE, unlike the halogen repo's advisory doc-check. That
    # one reports a judgement call; this one regenerates a derived file. If it
    # fails, _text/ would silently describe a card that no longer exists — and a
    # stale derived copy is worse than no copy, because it reads as current.
    print("⚠ runs on every commit; a failure blocks it, which is the point")
    return 0


def export_world_settings() -> int:
    """The gating, not the worlds — which lorebooks are active and how they fire.

    ⚠ A lorebook is INERT until something selects it, and the selection lives in
    settings.json, which is untracked. Versioning `worlds/*.json` without this
    captures the content and loses the wiring: a restore would hand back every
    lorebook with no record of which were global, which rode on a character, or
    what the token budget was. Small, non-secret, and it is the half that makes
    the other half mean something.
    """
    p = os.path.join(HERE, "settings.json")
    if not os.path.exists(p):
        return 0
    d = json.load(open(p, encoding="utf-8"))
    out = {"world_info_settings": d.get("world_info_settings")}
    pu = d.get("power_user") or {}
    # per-persona lorebook bindings, without dragging in the persona text again
    binds = {k: v.get("lorebook") for k, v in (pu.get("persona_descriptions") or {}).items()
             if isinstance(v, dict) and v.get("lorebook")}
    if binds:
        out["persona_lorebooks"] = binds
    with open(os.path.join(OUT, "world-settings.json"), "w", encoding="utf-8") as fh:
        json.dump(out, fh, indent=2, ensure_ascii=False, sort_keys=True)
        fh.write("\n")
    wis = (out.get("world_info_settings") or {}).get("world_info") or {}
    return len(wis.get("globalSelect") or [])


def main() -> int:
    if "--install-hook" in sys.argv:
        return install_hook()
    os.makedirs(OUT, exist_ok=True)
    cards = export_cards()
    personas = export_personas()
    active = export_world_settings()
    worlds = len([f for f in os.listdir(os.path.join(HERE, "worlds"))
                  if f.endswith(".json")]) if os.path.isdir(os.path.join(HERE, "worlds")) else 0
    print(f"  _text/cards/     {cards} card(s) extracted from PNG")
    print(f"  _text/personas.json  {personas} persona(s)")
    print(f"  worlds/          {worlds} lorebook(s) already JSON, tracked as-is")
    print(f"  _text/world-settings.json  gating captured; {active} world(s) globally active")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())