feat(ses): add SES light novel pipeline; extract shared lib/wiki.py

New pipeline at pipelines/ses/ publishes mikkeli/ses-light-novel to
the SES: MediaWiki namespace. All pages get Category:SES and
Category:SES:<type> (type from frontmatter, fallback to parent dir name).

Shared functions (connect_wiki, markdown_to_wikitext, strip_first_h1,
AUTO_BANNER_PREFIX) extracted from songs/publish.py into lib/wiki.py;
songs refactored to import from there.

Also adds publish-ses.yml workflow stub and updates Dockerfile and
root README.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitea/workflows/publish-ses.yml

@@ -0,0 +1,41 @@
+on:
+  workflow_call:
+    inputs:
+      SOURCE_REF:
+        description: "Commit SHA that was pushed to ses-light-novel"
+        required: true
+        type: string
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    container:
+      image: novoyuuparosk-wiki-runner:latest
+
+    steps:
+      - name: Checkout ses-light-novel
+        env:
+          FAPAT: ${{ secrets.FAPAT }}
+          URL_TO_GITEA: ${{ vars.URL_TO_GITEA }}
+        run: |
+          git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/mikkeli/ses-light-novel" ses-light-novel
+          git -C ses-light-novel checkout ${{ inputs.SOURCE_REF }}
+
+      - name: Checkout auto-wiki
+        env:
+          FAPAT: ${{ secrets.FAPAT }}
+          URL_TO_GITEA: ${{ vars.URL_TO_GITEA }}
+        run: git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/mikkeli/novoyuuparosk-auto-wiki" auto-wiki
+
+      - name: Publish
+        env:
+          WIKI_API_URL: ${{ vars.WIKI_API_URL }}
+          WIKI_BASE_URL: ${{ vars.WIKI_BASE_URL }}
+          WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }}
+          WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }}
+          SOURCE_REF: ${{ inputs.SOURCE_REF }}
+          GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/mikkeli/ses-light-novel
+        run: |
+          python auto-wiki/pipelines/ses/publish.py \
+            --source-dir ses-light-novel \
+            --all

Dockerfile

@@ -6,3 +6,6 @@ RUN apt-get update -qq \
 
 COPY pipelines/songs/requirements.txt /tmp/songs-requirements.txt
 RUN pip install --no-cache-dir -r /tmp/songs-requirements.txt
+
+COPY pipelines/ses/requirements.txt /tmp/ses-requirements.txt
+RUN pip install --no-cache-dir -r /tmp/ses-requirements.txt

README.md

@@ -7,6 +7,7 @@ CI/CD pipelines that auto-apply commits to https://wiki.novoyuuparosk.org from u
 | Path | Source repo | Purpose | Status |
 |---|---|---|---|
 | [`pipelines/songs/`](pipelines/songs/) | `mikkeli/ncmr-songs` | Song lyric pages | v1 live |
+| [`pipelines/ses/`](pipelines/ses/) | `mikkeli/ses-light-novel` | SES light novel pages | v1 in development |
 
 Per-pipeline READMEs cover everything specific to that pipeline (source schema, renderer, runtime, decisions). This root README covers only what's cross-cutting.
 

lib/wiki.py

@@ -0,0 +1,43 @@
+"""Shared utilities for novoyuuparosk-auto-wiki pipelines."""
+
+import os
+import subprocess
+from urllib.parse import urlparse
+
+import mwclient
+
+AUTO_BANNER_PREFIX = "{{Auto-generated"
+
+
+def strip_first_h1(text: str) -> str:
+    """Remove the first '# Heading' line and any immediately following blank line."""
+    lines = text.split("\n")
+    for i, line in enumerate(lines):
+        if line.strip().startswith("# "):
+            del lines[i]
+            if i < len(lines) and lines[i].strip() == "":
+                del lines[i]
+            break
+    return "\n".join(lines)
+
+
+def markdown_to_wikitext(body: str) -> str:
+    result = subprocess.run(
+        ["pandoc", "-f", "markdown", "-t", "mediawiki"],
+        input=body,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"pandoc failed: {result.stderr.strip()}")
+    return result.stdout
+
+
+def connect_wiki() -> mwclient.Site:
+    api_url = os.environ["WIKI_API_URL"]
+    parsed = urlparse(api_url)
+    host = parsed.netloc
+    path = parsed.path[: parsed.path.rfind("/") + 1] or "/"
+    site = mwclient.Site(host, path=path, scheme=parsed.scheme)
+    site.login(os.environ["WIKI_BOT_USER"], os.environ["WIKI_BOT_PASSWORD"])
+    return site

pipelines/ses/README.md

@@ -0,0 +1,84 @@
+# pipelines/ses
+
+Publishes SES light novel pages from `mikkeli/ses-light-novel` to `https://wiki.novoyuuparosk.org` under the `SES:` namespace.
+
+- Source-file contract (frontmatter and body conventions): [SCHEMA.md](SCHEMA.md)
+- Cross-cutting setup (wiki URL, bot identity, runner, Gitea secrets/variables): [repo root README](../../README.md)
+
+## v1 scope
+
+- YAML frontmatter parsing per [SCHEMA.md](SCHEMA.md)
+- `SES:` title prefix — all pages live in the SES MediaWiki namespace
+- Pandoc-based markdown → wikitext body rendering
+- First-line h1 stripping
+- Auto-generated banner (`{{Auto-generated|source=...|commit=<sha>}}`)
+- Category injection: `[[Category:SES]]` + `[[Category:SES:<type>]]` on every page; type from `type:` field, falling back to the file's parent directory name
+- Additional categories via optional `categories` list
+- MediaWiki bot API write with idempotency — no-op if wiki content matches generated output
+- Files without `wiki.publish: true` silently skipped
+- Pages with existing non-auto-generated content skipped (logged to stderr)
+
+Not in scope (intentional):
+
+- Sibling/cross-page link resolution — write wikilinks directly in the markdown body
+- Companion file uploads
+
+## Source repo and branch convention
+
+Source repo: `mikkeli/ses-light-novel`. The pipeline triggers on push to `master`.
+
+No branch convention enforced — work directly on `master` or use whatever branch workflow suits.
+
+**Excluded paths** (won't trigger the pipeline):
+
+- `README.md`, `.gitignore`, `.claude/**`
+
+## Invocation
+
+The pipeline runs as a self-contained Gitea Actions workflow in `mikkeli/ses-light-novel` (`.gitea/workflows/publish.yml`). It clones this repo at runtime to get the renderer.
+
+Triggers:
+- `push` to `master` (path-filtered as above)
+- `workflow_dispatch` — manual trigger from the Gitea Actions UI
+
+### Environment (secrets and variables)
+
+| Name | Kind | Source | Purpose |
+|---|---|---|---|
+| `WIKI_API_URL` | variable | Gitea (cross-cutting) | MediaWiki action API endpoint |
+| `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL |
+| `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login |
+| `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value |
+| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning |
+| `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos |
+| `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary |
+| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link |
+
+## Dependencies
+
+Job container: `novoyuuparosk-wiki-runner:latest` (pre-built, stored in the local Docker daemon on the runner host). Bakes in Python 3.12, Pandoc, and all pipeline Python packages — no install steps at job runtime.
+
+See the `Dockerfile` at the repo root and `requirements.txt` in this directory. Rebuild the image after changes to either.
+
+Shared code lives in `lib/wiki.py` (repo root).
+
+## One-off wiki setup
+
+- [ ] **`Template:Auto-generated`** — must exist on the wiki (shared with songs pipeline; already created).
+
+## Decisions log (ses pipeline)
+
+| Decision | Value | Date |
+|---|---|---|
+| Namespace | All pages prefixed `SES:` — auto-applied by renderer, not written in source `title:` | 2026-06-10 |
+| Type resolution | `type:` frontmatter field; falls back to immediate parent directory name | 2026-06-10 |
+| Categories | `Category:SES` + `Category:SES:<type>` on every page | 2026-06-10 |
+| Sibling links | Not implemented — write `[[SES:Page title]]` wikilinks directly in the body | 2026-06-10 |
+| WIP exclusion | None — `wiki.publish: false` is the only gate | 2026-06-10 |
+| Publish mode | Always `--all` | 2026-06-10 |
+| Idempotency | Commit SHA in banner, not timestamp | 2026-06-10 |
+| Manual-page protection | Bot skips pages without `{{Auto-generated` banner | 2026-06-10 |
+
+## Status
+
+v1 in development.

pipelines/ses/SCHEMA.md

@@ -0,0 +1,151 @@
+# SCHEMA — SES light novel source files
+
+Source-file contract for the SES pipeline. Files in `mikkeli/ses-light-novel` must follow this contract to be picked up by the auto-publisher.
+
+The schema lives entirely in the YAML frontmatter block at the top of each `.md` file. The body below the frontmatter is plain markdown, rendered to MediaWiki wikitext by Pandoc.
+
+## File layout
+
+```markdown
+---
+title: Wakkanai
+type: story
+wiki:
+  publish: true
+---
+
+## Section 1
+### The coast or the shore
+
+... body markdown ...
+```
+
+The renderer parses the frontmatter, validates required fields, strips the first `# Heading` line if present, then hands the body to Pandoc.
+
+## Fields
+
+### Required
+
+#### `title` (string)
+
+The MediaWiki page name, without the `SES:` prefix — the pipeline prepends it automatically. MediaWiki capitalises the first letter; lowercase input is fine.
+
+Example: `title: Wakkanai` → wiki page `SES:Wakkanai` at URL `/wiki/SES:Wakkanai`.
+
+#### `wiki.publish` (boolean)
+
+Publishing gate. `true` means the pipeline writes this file to the wiki. `false` means the file is ignored.
+
+`wiki` must be a YAML mapping, not a list:
+
+```yaml
+# correct
+wiki:
+  publish: true
+
+# wrong — will be silently skipped
+wiki:
+  - publish: true
+```
+
+### Optional
+
+#### `type` (string)
+
+The page type. Drives `[[Category:SES:<type>]]` injection. If omitted, the pipeline falls back to the file's immediate parent directory name (`stories`, `people`, `places`, etc.).
+
+Example: `type: story` → `[[Category:SES:story]]`.
+
+If neither `type` nor a meaningful directory name is available (file sits at the repo root), the fallback is `uncategorized`.
+
+#### `categories` (list of strings)
+
+Additional wiki categories to inject beyond the automatic `SES` and `SES:<type>` ones. Each entry becomes `[[Category:<value>]]`.
+
+```yaml
+categories:
+  - Featured
+```
+
+## Body
+
+Plain markdown. The renderer strips the first-line `# Heading` if present, then passes the body through Pandoc. Do not embed raw wikitext (`{{Template}}`, raw `[[Wikilinks]]`, etc.) unless you intend the literal output.
+
+## Renderer behaviour
+
+For each `.md` with `wiki.publish: true`:
+
+1. Parse and validate frontmatter.
+2. Strip the leading `# Heading` from the body if present.
+3. Pipe the body through `pandoc -f markdown -t mediawiki`.
+4. Prepend the auto-generated banner: `{{Auto-generated|source=<source URL>|commit=<sha>}}`.
+5. Append category tags: `[[Category:SES]]`, `[[Category:SES:<type>]]`, and any entries from `categories`.
+6. Read the current wiki page content via the MediaWiki API; if identical to the generated output, skip the write (idempotency).
+7. If the page exists and does not start with `{{Auto-generated`, skip with a warning (manual page protection).
+8. Otherwise, write the page with an edit summary referencing the source commit.
+
+## Implemented scope
+
+- Frontmatter parsing and validation
+- `SES:` title prefix
+- Pandoc-based markdown → wikitext body rendering
+- First-line h1 stripping
+- Banner template injection
+- `Category:SES` + `Category:SES:<type>` injection (type from frontmatter, fallback to parent directory)
+- Additional categories via `categories` list
+- Idempotent writes (no-op skip when content matches)
+- Manual-page protection (skip pages without auto-gen banner)
+- Files without `wiki.publish: true` silently skipped
+
+## Not in scope (intentional)
+
+- Sibling/cross-page link resolution — write wikilinks directly in the body
+- Separate upload of companion files (no LRC equivalent)
+
+## Examples
+
+### A story chapter
+
+```yaml
+---
+title: Wakkanai
+type: story
+wiki:
+  publish: true
+---
+```
+
+Wiki page: `SES:Wakkanai`. Categories: `SES`, `SES:story`.
+
+### A world entry (type from directory)
+
+```yaml
+---
+title: Emms White
+wiki:
+  publish: true
+---
+```
+
+File at `world/people/emms_white.md`. Type falls back to `people`.
+Wiki page: `SES:Emms White`. Categories: `SES`, `SES:people`.
+
+### A draft (not published)
+
+```yaml
+---
+title: Helsinki
+type: place
+wiki:
+  publish: false
+---
+```
+
+## Validation errors the renderer must produce
+
+Files without `wiki.publish: true` are silently skipped. The pipeline fails fast only on opted-in files with invalid data:
+
+- Missing or empty `title` on a file with `wiki.publish: true`
+- `wiki.publish` present but not a boolean
+- `categories` (if present) is not a list
+- Two publishable source files resolve to the same wiki title

pipelines/ses/publish.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""SES light novel pipeline — publishes .md files to MediaWiki under the SES: namespace."""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+import frontmatter
+import mwclient
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext, strip_first_h1
+
+
+def validate(fm: dict, path: Path) -> None:
+    errors = []
+
+    val = fm.get("title")
+    if not val or not str(val).strip():
+        errors.append("missing or empty 'title'")
+
+    if not isinstance(fm.get("wiki", {}).get("publish"), bool):
+        errors.append("'wiki.publish' must be a boolean (true/false), not a string")
+
+    if "categories" in fm and not isinstance(fm["categories"], list):
+        errors.append("'categories' must be a list")
+
+    if errors:
+        raise ValueError(f"{path}: " + "; ".join(errors))
+
+
+def get_page_title(fm: dict) -> str:
+    return f"SES:{fm['title']}"
+
+
+def get_page_type(fm: dict, path: Path, source_dir: Path) -> str:
+    if fm.get("type"):
+        return str(fm["type"])
+    rel = path.relative_to(source_dir)
+    parent = rel.parent.name
+    return parent if parent else "uncategorized"
+
+
+def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: str, page_type: str) -> str:
+    banner = f"{{{{Auto-generated|source={source_url}|commit={source_ref}}}}}"
+    cat_parts = [f"[[Category:SES]]", f"[[Category:SES:{page_type}]]"]
+    for cat in (fm.get("categories") or []):
+        cat_parts.append(f"[[Category:{cat}]]")
+    categories = "\n".join(cat_parts) + "\n"
+    return f"{banner}\n\n{body_wikitext}{categories}"
+
+
+def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, source_ref: str, gitea_repo_url: str) -> str:
+    """Returns 'published', 'noop', or 'skipped-manual'."""
+    rel = abs_path.relative_to(source_dir)
+    fm = post.metadata
+    title = get_page_title(fm)
+    page_type = get_page_type(fm, abs_path, source_dir)
+
+    body_wikitext = markdown_to_wikitext(strip_first_h1(post.content))
+    source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}"
+    page_content = build_wikitext(fm, body_wikitext, source_url, source_ref, page_type)
+
+    page = site.pages[title]
+    existing = page.text()
+
+    if existing and not existing.startswith(AUTO_BANNER_PREFIX):
+        return "skipped-manual"
+
+    if existing == page_content:
+        return "noop"
+
+    ref_short = source_ref[:8] if source_ref else "unknown"
+    page.save(page_content, summary=f"Auto-published from {ref_short} (ses pipeline)")
+    return "published"
+
+
+def load_and_validate(files: list[Path], source_dir: Path) -> dict[Path, object]:
+    posts = {}
+    titles: dict[str, Path] = {}
+
+    for path in files:
+        if not path.exists():
+            print(f"  warn: {path} not found, skipping", file=sys.stderr)
+            continue
+
+        post = frontmatter.load(str(path))
+
+        wiki = post.metadata.get("wiki")
+        if not isinstance(wiki, dict) or not wiki.get("publish"):
+            continue
+
+        validate(post.metadata, path)
+
+        title = get_page_title(post.metadata)
+        if title in titles:
+            raise ValueError(f"Duplicate wiki title '{title}': {path} and {titles[title]}")
+        titles[title] = path
+        posts[path] = post
+
+    return posts
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Publish SES light novel .md files to MediaWiki.")
+    parser.add_argument("--source-dir", required=True, type=Path)
+    parser.add_argument("--files", nargs="*", default=[], help="Relative paths within source-dir")
+    parser.add_argument("--all", action="store_true", help="Publish all publishable files")
+    args = parser.parse_args()
+
+    source_dir = args.source_dir.resolve()
+    source_ref = os.environ.get("SOURCE_REF", "")
+    gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/")
+
+    if args.all:
+        candidates = list(source_dir.rglob("*.md"))
+    elif args.files:
+        candidates = [source_dir / f for f in args.files]
+    else:
+        print("Nothing to do: pass --files or --all.")
+        return
+
+    files = [
+        p for p in candidates
+        if p.exists()
+        and p.suffix == ".md"
+        and p.name != "README.md"
+    ]
+
+    if not files:
+        print("No publishable candidates after filtering.")
+        return
+
+    print(f"Validating {len(files)} file(s)...")
+    posts = load_and_validate(files, source_dir)
+    print("Validation passed.")
+
+    site = connect_wiki()
+
+    counts = {"published": 0, "noop": 0, "skipped-manual": 0}
+    for abs_path, post in posts.items():
+        rel = abs_path.relative_to(source_dir)
+        outcome = publish_one(abs_path, source_dir, post, site, source_ref, gitea_repo_url)
+        counts[outcome] += 1
+        if outcome == "skipped-manual":
+            print(f"  [skipped-manual] {rel}  ← page exists without auto-gen banner; delete or add banner to hand over to bot", file=sys.stderr)
+        else:
+            print(f"  [{outcome}] {rel}")
+
+    print(
+        f"\nDone: {counts['published']} published, {counts['noop']} unchanged, "
+        f"{counts['skipped-manual']} skipped (existing manual pages)."
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"ERROR: {e}", file=sys.stderr)
+        sys.exit(1)

pipelines/ses/requirements.txt

@@ -0,0 +1,3 @@
+python-frontmatter
+mwclient
+PyYAML

pipelines/songs/publish.py

@@ -5,7 +5,6 @@ import argparse
 import hashlib
 import io
 import os
-import subprocess
 import sys
 from datetime import datetime
 from pathlib import Path
@@ -13,6 +12,9 @@ from pathlib import Path
 import frontmatter
 import mwclient
 
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext, strip_first_h1
+
 
 def validate(fm: dict, path: Path) -> None:
     errors = []
@@ -47,30 +49,6 @@ def validate(fm: dict, path: Path) -> None:
         raise ValueError(f"{path}: " + "; ".join(errors))
 
 
-def strip_first_h1(text: str) -> str:
-    """Remove the first '# Heading' line and any immediately following blank line."""
-    lines = text.split("\n")
-    for i, line in enumerate(lines):
-        if line.strip().startswith("# "):
-            del lines[i]
-            if i < len(lines) and lines[i].strip() == "":
-                del lines[i]
-            break
-    return "\n".join(lines)
-
-
-def markdown_to_wikitext(body: str) -> str:
-    result = subprocess.run(
-        ["pandoc", "-f", "markdown", "-t", "mediawiki"],
-        input=body,
-        capture_output=True,
-        text=True,
-    )
-    if result.returncode != 0:
-        raise RuntimeError(f"pandoc failed: {result.stderr.strip()}")
-    return result.stdout
-
-
 def build_sibling_map(posts: dict) -> dict[str, str]:
     """Map abs_path_str → wiki_title for all published posts."""
     return {str(path): str(post.metadata["title"]) for path, post in posts.items()}
@@ -115,19 +93,6 @@ def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: st
     return f"{banner}\n\n{body_wikitext}{lrc_line}{categories}"
 
 
-def connect_wiki() -> mwclient.Site:
-    from urllib.parse import urlparse
-
-    api_url = os.environ["WIKI_API_URL"]
-    parsed = urlparse(api_url)
-    host = parsed.netloc
-    # Strip api.php to get the wiki root path (e.g. "/" or "/w/")
-    path = parsed.path[: parsed.path.rfind("/") + 1] or "/"
-    site = mwclient.Site(host, path=path, scheme=parsed.scheme)
-    site.login(os.environ["WIKI_BOT_USER"], os.environ["WIKI_BOT_PASSWORD"])
-    return site
-
-
 def upload_lrc(lrc_path: Path, title: str, site: mwclient.Site) -> bool:
     """Upload LRC file to wiki if content has changed. Returns True if uploaded."""
     if not lrc_path.exists():
@@ -156,9 +121,6 @@ def upload_lrc(lrc_path: Path, title: str, site: mwclient.Site) -> bool:
     return True
 
 
-AUTO_BANNER_PREFIX = "{{Auto-generated"
-
-
 def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, source_ref: str, gitea_repo_url: str, sibling_map: dict[str, str]) -> str:
     """Returns 'published', 'noop', or 'skipped-manual'."""
     rel = abs_path.relative_to(source_dir)