diff --git a/.gitea/workflows/publish-ses.yml b/.gitea/workflows/publish-ses.yml new file mode 100644 index 0000000..4818cf5 --- /dev/null +++ b/.gitea/workflows/publish-ses.yml @@ -0,0 +1,41 @@ +on: + workflow_call: + inputs: + SOURCE_REF: + description: "Commit SHA that was pushed to ses-light-novel" + required: true + type: string + +jobs: + publish: + runs-on: ubuntu-latest + container: + image: novoyuuparosk-wiki-runner:latest + + steps: + - name: Checkout ses-light-novel + env: + FAPAT: ${{ secrets.FAPAT }} + URL_TO_GITEA: ${{ vars.URL_TO_GITEA }} + run: | + git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/mikkeli/ses-light-novel" ses-light-novel + git -C ses-light-novel checkout ${{ inputs.SOURCE_REF }} + + - name: Checkout auto-wiki + env: + FAPAT: ${{ secrets.FAPAT }} + URL_TO_GITEA: ${{ vars.URL_TO_GITEA }} + run: git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/mikkeli/novoyuuparosk-auto-wiki" auto-wiki + + - name: Publish + env: + WIKI_API_URL: ${{ vars.WIKI_API_URL }} + WIKI_BASE_URL: ${{ vars.WIKI_BASE_URL }} + WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} + WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} + SOURCE_REF: ${{ inputs.SOURCE_REF }} + GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/mikkeli/ses-light-novel + run: | + python auto-wiki/pipelines/ses/publish.py \ + --source-dir ses-light-novel \ + --all diff --git a/Dockerfile b/Dockerfile index ed3a301..77fd0a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,3 +6,6 @@ RUN apt-get update -qq \ COPY pipelines/songs/requirements.txt /tmp/songs-requirements.txt RUN pip install --no-cache-dir -r /tmp/songs-requirements.txt + +COPY pipelines/ses/requirements.txt /tmp/ses-requirements.txt +RUN pip install --no-cache-dir -r /tmp/ses-requirements.txt diff --git a/README.md b/README.md index 7e0d136..1d1de37 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ CI/CD pipelines that auto-apply commits to https://wiki.novoyuuparosk.org from u | Path | Source repo | Purpose | Status | |---|---|---|---| | [`pipelines/songs/`](pipelines/songs/) | `mikkeli/ncmr-songs` | Song lyric pages | v1 live | +| [`pipelines/ses/`](pipelines/ses/) | `mikkeli/ses-light-novel` | SES light novel pages | v1 in development | Per-pipeline READMEs cover everything specific to that pipeline (source schema, renderer, runtime, decisions). This root README covers only what's cross-cutting. diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/wiki.py b/lib/wiki.py new file mode 100644 index 0000000..65bab1f --- /dev/null +++ b/lib/wiki.py @@ -0,0 +1,43 @@ +"""Shared utilities for novoyuuparosk-auto-wiki pipelines.""" + +import os +import subprocess +from urllib.parse import urlparse + +import mwclient + +AUTO_BANNER_PREFIX = "{{Auto-generated" + + +def strip_first_h1(text: str) -> str: + """Remove the first '# Heading' line and any immediately following blank line.""" + lines = text.split("\n") + for i, line in enumerate(lines): + if line.strip().startswith("# "): + del lines[i] + if i < len(lines) and lines[i].strip() == "": + del lines[i] + break + return "\n".join(lines) + + +def markdown_to_wikitext(body: str) -> str: + result = subprocess.run( + ["pandoc", "-f", "markdown", "-t", "mediawiki"], + input=body, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"pandoc failed: {result.stderr.strip()}") + return result.stdout + + +def connect_wiki() -> mwclient.Site: + api_url = os.environ["WIKI_API_URL"] + parsed = urlparse(api_url) + host = parsed.netloc + path = parsed.path[: parsed.path.rfind("/") + 1] or "/" + site = mwclient.Site(host, path=path, scheme=parsed.scheme) + site.login(os.environ["WIKI_BOT_USER"], os.environ["WIKI_BOT_PASSWORD"]) + return site diff --git a/pipelines/ses/README.md b/pipelines/ses/README.md new file mode 100644 index 0000000..138f83c --- /dev/null +++ b/pipelines/ses/README.md @@ -0,0 +1,84 @@ +# pipelines/ses + +Publishes SES light novel pages from `mikkeli/ses-light-novel` to `https://wiki.novoyuuparosk.org` under the `SES:` namespace. + +- Source-file contract (frontmatter and body conventions): [SCHEMA.md](SCHEMA.md) +- Cross-cutting setup (wiki URL, bot identity, runner, Gitea secrets/variables): [repo root README](../../README.md) + +## v1 scope + +- YAML frontmatter parsing per [SCHEMA.md](SCHEMA.md) +- `SES:` title prefix — all pages live in the SES MediaWiki namespace +- Pandoc-based markdown → wikitext body rendering +- First-line h1 stripping +- Auto-generated banner (`{{Auto-generated|source=...|commit=}}`) +- Category injection: `[[Category:SES]]` + `[[Category:SES:]]` on every page; type from `type:` field, falling back to the file's parent directory name +- Additional categories via optional `categories` list +- MediaWiki bot API write with idempotency — no-op if wiki content matches generated output +- Files without `wiki.publish: true` silently skipped +- Pages with existing non-auto-generated content skipped (logged to stderr) + +Not in scope (intentional): + +- Sibling/cross-page link resolution — write wikilinks directly in the markdown body +- Companion file uploads + +## Source repo and branch convention + +Source repo: `mikkeli/ses-light-novel`. The pipeline triggers on push to `master`. + +No branch convention enforced — work directly on `master` or use whatever branch workflow suits. + +**Excluded paths** (won't trigger the pipeline): + +- `README.md`, `.gitignore`, `.claude/**` + +## Invocation + +The pipeline runs as a self-contained Gitea Actions workflow in `mikkeli/ses-light-novel` (`.gitea/workflows/publish.yml`). It clones this repo at runtime to get the renderer. + +Triggers: +- `push` to `master` (path-filtered as above) +- `workflow_dispatch` — manual trigger from the Gitea Actions UI + +### Environment (secrets and variables) + +| Name | Kind | Source | Purpose | +|---|---|---|---| +| `WIKI_API_URL` | variable | Gitea (cross-cutting) | MediaWiki action API endpoint | +| `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | +| `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login | +| `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | +| `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | +| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | + +## Dependencies + +Job container: `novoyuuparosk-wiki-runner:latest` (pre-built, stored in the local Docker daemon on the runner host). Bakes in Python 3.12, Pandoc, and all pipeline Python packages — no install steps at job runtime. + +See the `Dockerfile` at the repo root and `requirements.txt` in this directory. Rebuild the image after changes to either. + +Shared code lives in `lib/wiki.py` (repo root). + +## One-off wiki setup + +- [ ] **`Template:Auto-generated`** — must exist on the wiki (shared with songs pipeline; already created). + +## Decisions log (ses pipeline) + +| Decision | Value | Date | +|---|---|---| +| Namespace | All pages prefixed `SES:` — auto-applied by renderer, not written in source `title:` | 2026-06-10 | +| Type resolution | `type:` frontmatter field; falls back to immediate parent directory name | 2026-06-10 | +| Categories | `Category:SES` + `Category:SES:` on every page | 2026-06-10 | +| Sibling links | Not implemented — write `[[SES:Page title]]` wikilinks directly in the body | 2026-06-10 | +| WIP exclusion | None — `wiki.publish: false` is the only gate | 2026-06-10 | +| Publish mode | Always `--all` | 2026-06-10 | +| Idempotency | Commit SHA in banner, not timestamp | 2026-06-10 | +| Manual-page protection | Bot skips pages without `{{Auto-generated` banner | 2026-06-10 | + +## Status + +v1 in development. diff --git a/pipelines/ses/SCHEMA.md b/pipelines/ses/SCHEMA.md new file mode 100644 index 0000000..d5c78a4 --- /dev/null +++ b/pipelines/ses/SCHEMA.md @@ -0,0 +1,151 @@ +# SCHEMA — SES light novel source files + +Source-file contract for the SES pipeline. Files in `mikkeli/ses-light-novel` must follow this contract to be picked up by the auto-publisher. + +The schema lives entirely in the YAML frontmatter block at the top of each `.md` file. The body below the frontmatter is plain markdown, rendered to MediaWiki wikitext by Pandoc. + +## File layout + +```markdown +--- +title: Wakkanai +type: story +wiki: + publish: true +--- + +## Section 1 +### The coast or the shore + +... body markdown ... +``` + +The renderer parses the frontmatter, validates required fields, strips the first `# Heading` line if present, then hands the body to Pandoc. + +## Fields + +### Required + +#### `title` (string) + +The MediaWiki page name, without the `SES:` prefix — the pipeline prepends it automatically. MediaWiki capitalises the first letter; lowercase input is fine. + +Example: `title: Wakkanai` → wiki page `SES:Wakkanai` at URL `/wiki/SES:Wakkanai`. + +#### `wiki.publish` (boolean) + +Publishing gate. `true` means the pipeline writes this file to the wiki. `false` means the file is ignored. + +`wiki` must be a YAML mapping, not a list: + +```yaml +# correct +wiki: + publish: true + +# wrong — will be silently skipped +wiki: + - publish: true +``` + +### Optional + +#### `type` (string) + +The page type. Drives `[[Category:SES:]]` injection. If omitted, the pipeline falls back to the file's immediate parent directory name (`stories`, `people`, `places`, etc.). + +Example: `type: story` → `[[Category:SES:story]]`. + +If neither `type` nor a meaningful directory name is available (file sits at the repo root), the fallback is `uncategorized`. + +#### `categories` (list of strings) + +Additional wiki categories to inject beyond the automatic `SES` and `SES:` ones. Each entry becomes `[[Category:]]`. + +```yaml +categories: + - Featured +``` + +## Body + +Plain markdown. The renderer strips the first-line `# Heading` if present, then passes the body through Pandoc. Do not embed raw wikitext (`{{Template}}`, raw `[[Wikilinks]]`, etc.) unless you intend the literal output. + +## Renderer behaviour + +For each `.md` with `wiki.publish: true`: + +1. Parse and validate frontmatter. +2. Strip the leading `# Heading` from the body if present. +3. Pipe the body through `pandoc -f markdown -t mediawiki`. +4. Prepend the auto-generated banner: `{{Auto-generated|source=|commit=}}`. +5. Append category tags: `[[Category:SES]]`, `[[Category:SES:]]`, and any entries from `categories`. +6. Read the current wiki page content via the MediaWiki API; if identical to the generated output, skip the write (idempotency). +7. If the page exists and does not start with `{{Auto-generated`, skip with a warning (manual page protection). +8. Otherwise, write the page with an edit summary referencing the source commit. + +## Implemented scope + +- Frontmatter parsing and validation +- `SES:` title prefix +- Pandoc-based markdown → wikitext body rendering +- First-line h1 stripping +- Banner template injection +- `Category:SES` + `Category:SES:` injection (type from frontmatter, fallback to parent directory) +- Additional categories via `categories` list +- Idempotent writes (no-op skip when content matches) +- Manual-page protection (skip pages without auto-gen banner) +- Files without `wiki.publish: true` silently skipped + +## Not in scope (intentional) + +- Sibling/cross-page link resolution — write wikilinks directly in the body +- Separate upload of companion files (no LRC equivalent) + +## Examples + +### A story chapter + +```yaml +--- +title: Wakkanai +type: story +wiki: + publish: true +--- +``` + +Wiki page: `SES:Wakkanai`. Categories: `SES`, `SES:story`. + +### A world entry (type from directory) + +```yaml +--- +title: Emms White +wiki: + publish: true +--- +``` + +File at `world/people/emms_white.md`. Type falls back to `people`. +Wiki page: `SES:Emms White`. Categories: `SES`, `SES:people`. + +### A draft (not published) + +```yaml +--- +title: Helsinki +type: place +wiki: + publish: false +--- +``` + +## Validation errors the renderer must produce + +Files without `wiki.publish: true` are silently skipped. The pipeline fails fast only on opted-in files with invalid data: + +- Missing or empty `title` on a file with `wiki.publish: true` +- `wiki.publish` present but not a boolean +- `categories` (if present) is not a list +- Two publishable source files resolve to the same wiki title diff --git a/pipelines/ses/publish.py b/pipelines/ses/publish.py new file mode 100644 index 0000000..179089c --- /dev/null +++ b/pipelines/ses/publish.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""SES light novel pipeline — publishes .md files to MediaWiki under the SES: namespace.""" + +import argparse +import os +import sys +from pathlib import Path + +import frontmatter +import mwclient + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext, strip_first_h1 + + +def validate(fm: dict, path: Path) -> None: + errors = [] + + val = fm.get("title") + if not val or not str(val).strip(): + errors.append("missing or empty 'title'") + + if not isinstance(fm.get("wiki", {}).get("publish"), bool): + errors.append("'wiki.publish' must be a boolean (true/false), not a string") + + if "categories" in fm and not isinstance(fm["categories"], list): + errors.append("'categories' must be a list") + + if errors: + raise ValueError(f"{path}: " + "; ".join(errors)) + + +def get_page_title(fm: dict) -> str: + return f"SES:{fm['title']}" + + +def get_page_type(fm: dict, path: Path, source_dir: Path) -> str: + if fm.get("type"): + return str(fm["type"]) + rel = path.relative_to(source_dir) + parent = rel.parent.name + return parent if parent else "uncategorized" + + +def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: str, page_type: str) -> str: + banner = f"{{{{Auto-generated|source={source_url}|commit={source_ref}}}}}" + cat_parts = [f"[[Category:SES]]", f"[[Category:SES:{page_type}]]"] + for cat in (fm.get("categories") or []): + cat_parts.append(f"[[Category:{cat}]]") + categories = "\n".join(cat_parts) + "\n" + return f"{banner}\n\n{body_wikitext}{categories}" + + +def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, source_ref: str, gitea_repo_url: str) -> str: + """Returns 'published', 'noop', or 'skipped-manual'.""" + rel = abs_path.relative_to(source_dir) + fm = post.metadata + title = get_page_title(fm) + page_type = get_page_type(fm, abs_path, source_dir) + + body_wikitext = markdown_to_wikitext(strip_first_h1(post.content)) + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + page_content = build_wikitext(fm, body_wikitext, source_url, source_ref, page_type) + + page = site.pages[title] + existing = page.text() + + if existing and not existing.startswith(AUTO_BANNER_PREFIX): + return "skipped-manual" + + if existing == page_content: + return "noop" + + ref_short = source_ref[:8] if source_ref else "unknown" + page.save(page_content, summary=f"Auto-published from {ref_short} (ses pipeline)") + return "published" + + +def load_and_validate(files: list[Path], source_dir: Path) -> dict[Path, object]: + posts = {} + titles: dict[str, Path] = {} + + for path in files: + if not path.exists(): + print(f" warn: {path} not found, skipping", file=sys.stderr) + continue + + post = frontmatter.load(str(path)) + + wiki = post.metadata.get("wiki") + if not isinstance(wiki, dict) or not wiki.get("publish"): + continue + + validate(post.metadata, path) + + title = get_page_title(post.metadata) + if title in titles: + raise ValueError(f"Duplicate wiki title '{title}': {path} and {titles[title]}") + titles[title] = path + posts[path] = post + + return posts + + +def main() -> None: + parser = argparse.ArgumentParser(description="Publish SES light novel .md files to MediaWiki.") + parser.add_argument("--source-dir", required=True, type=Path) + parser.add_argument("--files", nargs="*", default=[], help="Relative paths within source-dir") + parser.add_argument("--all", action="store_true", help="Publish all publishable files") + args = parser.parse_args() + + source_dir = args.source_dir.resolve() + source_ref = os.environ.get("SOURCE_REF", "") + gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + + if args.all: + candidates = list(source_dir.rglob("*.md")) + elif args.files: + candidates = [source_dir / f for f in args.files] + else: + print("Nothing to do: pass --files or --all.") + return + + files = [ + p for p in candidates + if p.exists() + and p.suffix == ".md" + and p.name != "README.md" + ] + + if not files: + print("No publishable candidates after filtering.") + return + + print(f"Validating {len(files)} file(s)...") + posts = load_and_validate(files, source_dir) + print("Validation passed.") + + site = connect_wiki() + + counts = {"published": 0, "noop": 0, "skipped-manual": 0} + for abs_path, post in posts.items(): + rel = abs_path.relative_to(source_dir) + outcome = publish_one(abs_path, source_dir, post, site, source_ref, gitea_repo_url) + counts[outcome] += 1 + if outcome == "skipped-manual": + print(f" [skipped-manual] {rel} ← page exists without auto-gen banner; delete or add banner to hand over to bot", file=sys.stderr) + else: + print(f" [{outcome}] {rel}") + + print( + f"\nDone: {counts['published']} published, {counts['noop']} unchanged, " + f"{counts['skipped-manual']} skipped (existing manual pages)." + ) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) diff --git a/pipelines/ses/requirements.txt b/pipelines/ses/requirements.txt new file mode 100644 index 0000000..09f67a7 --- /dev/null +++ b/pipelines/ses/requirements.txt @@ -0,0 +1,3 @@ +python-frontmatter +mwclient +PyYAML diff --git a/pipelines/songs/publish.py b/pipelines/songs/publish.py index f87cc78..8272be2 100644 --- a/pipelines/songs/publish.py +++ b/pipelines/songs/publish.py @@ -5,7 +5,6 @@ import argparse import hashlib import io import os -import subprocess import sys from datetime import datetime from pathlib import Path @@ -13,6 +12,9 @@ from pathlib import Path import frontmatter import mwclient +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext, strip_first_h1 + def validate(fm: dict, path: Path) -> None: errors = [] @@ -47,30 +49,6 @@ def validate(fm: dict, path: Path) -> None: raise ValueError(f"{path}: " + "; ".join(errors)) -def strip_first_h1(text: str) -> str: - """Remove the first '# Heading' line and any immediately following blank line.""" - lines = text.split("\n") - for i, line in enumerate(lines): - if line.strip().startswith("# "): - del lines[i] - if i < len(lines) and lines[i].strip() == "": - del lines[i] - break - return "\n".join(lines) - - -def markdown_to_wikitext(body: str) -> str: - result = subprocess.run( - ["pandoc", "-f", "markdown", "-t", "mediawiki"], - input=body, - capture_output=True, - text=True, - ) - if result.returncode != 0: - raise RuntimeError(f"pandoc failed: {result.stderr.strip()}") - return result.stdout - - def build_sibling_map(posts: dict) -> dict[str, str]: """Map abs_path_str → wiki_title for all published posts.""" return {str(path): str(post.metadata["title"]) for path, post in posts.items()} @@ -115,19 +93,6 @@ def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: st return f"{banner}\n\n{body_wikitext}{lrc_line}{categories}" -def connect_wiki() -> mwclient.Site: - from urllib.parse import urlparse - - api_url = os.environ["WIKI_API_URL"] - parsed = urlparse(api_url) - host = parsed.netloc - # Strip api.php to get the wiki root path (e.g. "/" or "/w/") - path = parsed.path[: parsed.path.rfind("/") + 1] or "/" - site = mwclient.Site(host, path=path, scheme=parsed.scheme) - site.login(os.environ["WIKI_BOT_USER"], os.environ["WIKI_BOT_PASSWORD"]) - return site - - def upload_lrc(lrc_path: Path, title: str, site: mwclient.Site) -> bool: """Upload LRC file to wiki if content has changed. Returns True if uploaded.""" if not lrc_path.exists(): @@ -156,9 +121,6 @@ def upload_lrc(lrc_path: Path, title: str, site: mwclient.Site) -> bool: return True -AUTO_BANNER_PREFIX = "{{Auto-generated" - - def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, source_ref: str, gitea_repo_url: str, sibling_map: dict[str, str]) -> str: """Returns 'published', 'noop', or 'skipped-manual'.""" rel = abs_path.relative_to(source_dir)