diff --git a/.gitea/workflows/publish-tech.yml b/.gitea/workflows/publish-tech.yml new file mode 100644 index 0000000..8728896 --- /dev/null +++ b/.gitea/workflows/publish-tech.yml @@ -0,0 +1,41 @@ +on: + workflow_call: + inputs: + SOURCE_REF: + description: "Commit SHA that was pushed to tech-blogs" + required: true + type: string + +jobs: + publish: + runs-on: ubuntu-latest + container: + image: novoyuuparosk-wiki-runner:latest + + steps: + - name: Checkout tech-blogs + env: + FAPAT: ${{ secrets.FAPAT }} + URL_TO_GITEA: ${{ vars.URL_TO_GITEA }} + run: | + git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/mikkeli/tech-blogs" tech-blogs + git -C tech-blogs checkout ${{ inputs.SOURCE_REF }} + + - name: Checkout auto-wiki + env: + FAPAT: ${{ secrets.FAPAT }} + URL_TO_GITEA: ${{ vars.URL_TO_GITEA }} + run: git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/mikkeli/novoyuuparosk-auto-wiki" auto-wiki + + - name: Publish + env: + WIKI_API_URL: ${{ vars.WIKI_API_URL }} + WIKI_BASE_URL: ${{ vars.WIKI_BASE_URL }} + WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} + WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} + SOURCE_REF: ${{ inputs.SOURCE_REF }} + GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/mikkeli/tech-blogs + run: | + python auto-wiki/pipelines/tech/publish.py \ + --source-dir tech-blogs \ + --all diff --git a/Dockerfile b/Dockerfile index 77fd0a7..3cf7df1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,3 +9,6 @@ RUN pip install --no-cache-dir -r /tmp/songs-requirements.txt COPY pipelines/ses/requirements.txt /tmp/ses-requirements.txt RUN pip install --no-cache-dir -r /tmp/ses-requirements.txt + +COPY pipelines/tech/requirements.txt /tmp/tech-requirements.txt +RUN pip install --no-cache-dir -r /tmp/tech-requirements.txt diff --git a/README.md b/README.md index 1d1de37..46e766a 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ CI/CD pipelines that auto-apply commits to https://wiki.novoyuuparosk.org from u |---|---|---|---| | [`pipelines/songs/`](pipelines/songs/) | `mikkeli/ncmr-songs` | Song lyric pages | v1 live | | [`pipelines/ses/`](pipelines/ses/) | `mikkeli/ses-light-novel` | SES light novel pages | v1 in development | +| [`pipelines/tech/`](pipelines/tech/) | `mikkeli/tech-blogs` | Tech blog posts | v1 ready (source repo not yet initialised) | Per-pipeline READMEs cover everything specific to that pipeline (source schema, renderer, runtime, decisions). This root README covers only what's cross-cutting. diff --git a/pipelines/tech/README.md b/pipelines/tech/README.md new file mode 100644 index 0000000..2f9523e --- /dev/null +++ b/pipelines/tech/README.md @@ -0,0 +1,79 @@ +# pipelines/tech + +Publishes tech blog posts from `mikkeli/tech-blogs` to `https://wiki.novoyuuparosk.org`. + +- Source-file contract (frontmatter and body conventions): [SCHEMA.md](SCHEMA.md) +- Cross-cutting setup (wiki URL, bot identity, runner, Gitea secrets/variables): [repo root README](../../README.md) + +## v1 scope + +- YAML frontmatter parsing per [SCHEMA.md](SCHEMA.md) +- Pandoc-based markdown → wikitext body rendering +- First-line h1 stripping +- Auto-generated banner (`{{Auto-generated|source=...|commit=}}`) +- Category injection: `[[Category:Tech blog]]` on every page; `[[Category:Blog:]]` from `date` field; `[[Category:]]` per tag entry +- No title prefix — pages live at their bare title +- MediaWiki bot API write with idempotency — no-op if wiki content matches generated output +- Files without `wiki.publish: true` silently skipped +- Pages with existing non-auto-generated content skipped (logged to stderr) + +## Source repo + +Source repo: `mikkeli/tech-blogs` (not yet initialised — pipeline code is ready). + +The pipeline triggers on push to `master` and supports `workflow_dispatch` for manual runs. + +**Excluded paths** (won't trigger the pipeline): + +- `README.md`, `.gitignore`, `.claude/**` + +## Invocation + +The pipeline runs as a self-contained Gitea Actions workflow in `mikkeli/tech-blogs` (`.gitea/workflows/publish.yml`). It clones this repo at runtime to get the renderer. + +Triggers: +- `push` to `master` (path-filtered as above) +- `workflow_dispatch` — manual trigger from the Gitea Actions UI + +### Environment (secrets and variables) + +| Name | Kind | Source | Purpose | +|---|---|---|---| +| `WIKI_API_URL` | variable | Gitea (cross-cutting) | MediaWiki action API endpoint | +| `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | +| `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login | +| `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | +| `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | +| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | + +## Dependencies + +Job container: `novoyuuparosk-wiki-runner:latest` (pre-built, stored in the local Docker daemon on the runner host). Bakes in Python 3.12, Pandoc, and all pipeline Python packages — no install steps at job runtime. + +See the `Dockerfile` at the repo root and `requirements.txt` in this directory. Rebuild the image after changes to either. + +Shared code lives in `lib/wiki.py` (repo root). + +## One-off wiki setup + +- [ ] **`Template:Auto-generated`** — must exist on the wiki (shared with all pipelines; already created). + +## Decisions log (tech pipeline) + +| Decision | Value | Date | +|---|---|---| +| No title prefix | Tech posts live at bare titles — no namespace needed | 2026-06-10 | +| Fixed category | Every post gets `Category:Tech blog` | 2026-06-10 | +| Year category | `Category:Blog:` from `date` field; omitted if no date | 2026-06-10 | +| Tag categories | Each tag → `[[Category:]]` with no prefix | 2026-06-10 | +| WIP exclusion | None — `wiki.publish: false` is the only gate | 2026-06-10 | +| Publish mode | Always `--all` | 2026-06-10 | +| Idempotency | Commit SHA in banner | 2026-06-10 | +| Manual-page protection | Bot skips pages without `{{Auto-generated` banner | 2026-06-10 | +| Source repo | `mikkeli/tech-blogs` — pipeline ships before repo is initialised | 2026-06-10 | + +## Status + +v1 pipeline ready. Source repo (`mikkeli/tech-blogs`) not yet initialised. diff --git a/pipelines/tech/SCHEMA.md b/pipelines/tech/SCHEMA.md new file mode 100644 index 0000000..baa5867 --- /dev/null +++ b/pipelines/tech/SCHEMA.md @@ -0,0 +1,131 @@ +# SCHEMA — tech blog source files + +Source-file contract for the tech blog pipeline. Files in `mikkeli/tech-blogs` must follow this contract to be picked up by the auto-publisher. + +The schema lives entirely in the YAML frontmatter block at the top of each `.md` file. The body is plain markdown, rendered to MediaWiki wikitext by Pandoc. + +## File layout + +```markdown +--- +title: Building a Raspberry Pi Kubernetes cluster +date: 2026-06-10 +tags: + - Raspberry Pi + - Kubernetes + - homelab +wiki: + publish: true +--- + +# Building a Raspberry Pi Kubernetes cluster + +... body markdown ... +``` + +## Fields + +### Required + +#### `title` (string) + +The MediaWiki page name. MediaWiki capitalises the first letter; lowercase input is fine. No prefix is applied — the title is used as-is. + +#### `wiki.publish` (boolean) + +Publishing gate. `true` means the pipeline writes this file to the wiki. `false` (or absent) means the file is ignored. `wiki` must be a YAML mapping: + +```yaml +# correct +wiki: + publish: true +``` + +### Optional + +#### `date` (ISO 8601 date) + +Publication date in `YYYY-MM-DD` format. Drives `[[Category:Blog:]]` injection. If omitted, no year category is added. + +#### `tags` (list of strings) + +Topic tags. Each entry becomes `[[Category:]]` at the bottom of the rendered page. Tags are used as-is — no `Blog:` prefix is added. + +```yaml +tags: + - Raspberry Pi + - Kubernetes + - homelab +``` + +## Body + +Plain markdown. The renderer strips the first-line `# Heading` if present (it duplicates the wiki page title), then passes the body through Pandoc. + +## Renderer behaviour + +For each `.md` with `wiki.publish: true`: + +1. Parse and validate frontmatter. +2. Strip the leading `# Heading` from the body if present. +3. Pipe the body through `pandoc -f markdown -t mediawiki`. +4. Prepend the auto-generated banner: `{{Auto-generated|source=|commit=}}`. +5. Append category tags: `[[Category:Tech blog]]`; `[[Category:Blog:]]` if `date` is set; `[[Category:]]` for each tag. +6. Read the current wiki page content; if identical, skip the write (idempotency). +7. If the page exists without `{{Auto-generated`, skip with a warning (manual page protection). +8. Otherwise, write with an edit summary referencing the source commit. + +## Implemented scope + +- Frontmatter parsing and validation +- Pandoc-based markdown → wikitext rendering +- First-line h1 stripping +- Banner template injection +- `Category:Tech blog` on every published page +- `Category:Blog:` from `date` field +- `Category:` per entry in `tags` +- Idempotent writes +- Manual-page protection +- Files without `wiki.publish: true` silently skipped + +## Not in scope (intentional) + +- Namespace prefix — posts live at bare titles +- Series grouping — use tags for now + +## Examples + +### A published post + +```yaml +--- +title: Running act_runner in Docker on a Raspberry Pi +date: 2026-06-01 +tags: + - Raspberry Pi + - Gitea + - CI/CD +wiki: + publish: true +--- +``` + +Categories: `Tech blog`, `Blog:2026`, `Raspberry Pi`, `Gitea`, `CI/CD`. + +### A draft + +```yaml +--- +title: Notes on aarch64 cross-compilation +wiki: + publish: false +--- +``` + +## Validation errors the renderer must produce + +- Missing or empty `title` on a file with `wiki.publish: true` +- `wiki.publish` present but not a boolean +- `date` (if present) does not parse as `YYYY-MM-DD` +- `tags` (if present) is not a list +- Two publishable files declare the same `title` diff --git a/pipelines/tech/publish.py b/pipelines/tech/publish.py new file mode 100644 index 0000000..7fc5d91 --- /dev/null +++ b/pipelines/tech/publish.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Tech blog pipeline — publishes .md files to MediaWiki.""" + +import argparse +import os +import sys +from datetime import datetime +from pathlib import Path + +import frontmatter +import mwclient + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext, strip_first_h1 + + +def validate(fm: dict, path: Path) -> None: + errors = [] + + val = fm.get("title") + if not val or not str(val).strip(): + errors.append("missing or empty 'title'") + + if not isinstance(fm.get("wiki", {}).get("publish"), bool): + errors.append("'wiki.publish' must be a boolean (true/false), not a string") + + if "date" in fm and fm["date"]: + try: + datetime.strptime(str(fm["date"]), "%Y-%m-%d") + except ValueError: + errors.append("'date' must be YYYY-MM-DD") + + if "tags" in fm and not isinstance(fm["tags"], list): + errors.append("'tags' must be a list") + + if errors: + raise ValueError(f"{path}: " + "; ".join(errors)) + + +def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: str) -> str: + banner = f"{{{{Auto-generated|source={source_url}|commit={source_ref}}}}}" + cat_parts = ["[[Category:Tech blog]]"] + date_val = fm.get("date") + if date_val: + year = str(date_val)[:4] + cat_parts.append(f"[[Category:Blog:{year}]]") + for tag in (fm.get("tags") or []): + cat_parts.append(f"[[Category:{tag}]]") + categories = "\n".join(cat_parts) + "\n" + return f"{banner}\n\n{body_wikitext}{categories}" + + +def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, source_ref: str, gitea_repo_url: str) -> str: + """Returns 'published', 'noop', or 'skipped-manual'.""" + rel = abs_path.relative_to(source_dir) + fm = post.metadata + title = str(fm["title"]) + + body_wikitext = markdown_to_wikitext(strip_first_h1(post.content)) + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + page_content = build_wikitext(fm, body_wikitext, source_url, source_ref) + + page = site.pages[title] + existing = page.text() + + if existing and not existing.startswith(AUTO_BANNER_PREFIX): + return "skipped-manual" + + if existing == page_content: + return "noop" + + ref_short = source_ref[:8] if source_ref else "unknown" + page.save(page_content, summary=f"Auto-published from {ref_short} (tech pipeline)") + return "published" + + +def load_and_validate(files: list[Path], source_dir: Path) -> dict[Path, object]: + posts = {} + titles: dict[str, Path] = {} + + for path in files: + if not path.exists(): + print(f" warn: {path} not found, skipping", file=sys.stderr) + continue + + post = frontmatter.load(str(path)) + + wiki = post.metadata.get("wiki") + if not isinstance(wiki, dict) or not wiki.get("publish"): + continue + + validate(post.metadata, path) + + title = str(post.metadata["title"]) + if title in titles: + raise ValueError(f"Duplicate wiki title '{title}': {path} and {titles[title]}") + titles[title] = path + posts[path] = post + + return posts + + +def main() -> None: + parser = argparse.ArgumentParser(description="Publish tech blog .md files to MediaWiki.") + parser.add_argument("--source-dir", required=True, type=Path) + parser.add_argument("--files", nargs="*", default=[], help="Relative paths within source-dir") + parser.add_argument("--all", action="store_true", help="Publish all publishable files") + args = parser.parse_args() + + source_dir = args.source_dir.resolve() + source_ref = os.environ.get("SOURCE_REF", "") + gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + + if args.all: + candidates = list(source_dir.rglob("*.md")) + elif args.files: + candidates = [source_dir / f for f in args.files] + else: + print("Nothing to do: pass --files or --all.") + return + + files = [ + p for p in candidates + if p.exists() + and p.suffix == ".md" + and p.name != "README.md" + ] + + if not files: + print("No publishable candidates after filtering.") + return + + print(f"Validating {len(files)} file(s)...") + posts = load_and_validate(files, source_dir) + print("Validation passed.") + + site = connect_wiki() + + counts = {"published": 0, "noop": 0, "skipped-manual": 0} + for abs_path, post in posts.items(): + rel = abs_path.relative_to(source_dir) + outcome = publish_one(abs_path, source_dir, post, site, source_ref, gitea_repo_url) + counts[outcome] += 1 + if outcome == "skipped-manual": + print(f" [skipped-manual] {rel} ← page exists without auto-gen banner; delete or add banner to hand over to bot", file=sys.stderr) + else: + print(f" [{outcome}] {rel}") + + print( + f"\nDone: {counts['published']} published, {counts['noop']} unchanged, " + f"{counts['skipped-manual']} skipped (existing manual pages)." + ) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) diff --git a/pipelines/tech/requirements.txt b/pipelines/tech/requirements.txt new file mode 100644 index 0000000..09f67a7 --- /dev/null +++ b/pipelines/tech/requirements.txt @@ -0,0 +1,3 @@ +python-frontmatter +mwclient +PyYAML