diff --git a/.gitea/workflows/publish-songs.yml b/.gitea/workflows/publish-songs.yml index ece89cf..29b46fe 100644 --- a/.gitea/workflows/publish-songs.yml +++ b/.gitea/workflows/publish-songs.yml @@ -6,14 +6,9 @@ on: required: true type: string SOURCE_BASE_REF: - description: "Commit SHA before the push (all-zeros = first push)" + description: "Commit SHA before the push (empty or all-zeros = first push)" required: true type: string - FULL_PUBLISH: - description: "Re-publish every file regardless of diff" - required: false - type: boolean - default: false jobs: publish: @@ -26,9 +21,10 @@ jobs: env: FAPAT: ${{ secrets.FAPAT }} URL_TO_GITEA: ${{ vars.URL_TO_GITEA }} + SOURCE_REF: ${{ inputs.SOURCE_REF }} run: | git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/novoyuuparosk-wiki/ncmr-songs" ncmr-songs - git -C ncmr-songs checkout ${{ inputs.SOURCE_REF }} + git -C ncmr-songs checkout "$SOURCE_REF" - name: Checkout auto-wiki env: @@ -36,24 +32,28 @@ jobs: URL_TO_GITEA: ${{ vars.URL_TO_GITEA }} run: git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/novoyuuparosk-wiki/novoyuuparosk-auto-wiki" auto-wiki + # The diff bounds what gets *written*, not what gets *read*. The publisher + # always scans the whole repository to resolve links and page titles, then + # writes only these files (plus, one hop out, any page they link to or that + # links to them). Passing a subset here is safe for that reason and only + # that reason — resolution no longer depends on the batch. - name: Detect changed files id: diff + env: + BASE: ${{ inputs.SOURCE_BASE_REF }} + HEAD: ${{ inputs.SOURCE_REF }} run: | - BASE="${{ inputs.SOURCE_BASE_REF }}" - HEAD="${{ inputs.SOURCE_REF }}" - # All-zeros base means first push — publish everything + # Empty or all-zeros base means first push or manual dispatch — publish everything if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then + echo "First push or manual dispatch — publishing everything." echo "mode=all" >> "$GITHUB_OUTPUT" - else - git -C ncmr-songs diff --name-only "$BASE" "$HEAD" -- '*.md' \ - | grep -v '^wip/' \ - | grep -v '^README\.md' \ - > /tmp/changed.txt 2>/dev/null || true - echo "Changed files:" - cat /tmp/changed.txt - echo "mode=incremental" >> "$GITHUB_OUTPUT" - echo "files=$(tr '\n' ' ' < /tmp/changed.txt)" >> "$GITHUB_OUTPUT" + exit 0 fi + CHANGED=$(git -C ncmr-songs diff --name-only "$BASE" "$HEAD" -- '*.md' | grep -v '^wip/' || true) + echo "Changed files:" + echo "$CHANGED" + echo "mode=incremental" >> "$GITHUB_OUTPUT" + echo "files=$(echo "$CHANGED" | tr '\n' ' ')" >> "$GITHUB_OUTPUT" - name: Publish env: @@ -63,20 +63,14 @@ jobs: WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} SOURCE_REF: ${{ inputs.SOURCE_REF }} GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/novoyuuparosk-wiki/ncmr-songs + MODE: ${{ steps.diff.outputs.mode }} + FILES: ${{ steps.diff.outputs.files }} run: | - MODE="${{ steps.diff.outputs.mode }}" - if [ "$MODE" = "all" ] || [ "${{ inputs.FULL_PUBLISH }}" = "true" ]; then - python auto-wiki/pipelines/songs/publish.py \ - --source-dir ncmr-songs \ - --all - else - FILES="${{ steps.diff.outputs.files }}" - if [ -z "$FILES" ]; then - echo "No changed .md files detected. Nothing to publish." - exit 0 - fi + if [ "$MODE" = "all" ]; then + python auto-wiki/pipelines/songs/publish.py --source-dir ncmr-songs --all + elif [ -n "$(echo "$FILES" | tr -d '[:space:]')" ]; then # shellcheck disable=SC2086 - python auto-wiki/pipelines/songs/publish.py \ - --source-dir ncmr-songs \ - --files $FILES + python auto-wiki/pipelines/songs/publish.py --source-dir ncmr-songs --files $FILES + else + echo "No changed .md files outside wip/. Nothing to publish." fi diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml new file mode 100644 index 0000000..78243b5 --- /dev/null +++ b/.gitea/workflows/test.yml @@ -0,0 +1,30 @@ +name: tests + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + pytest: + runs-on: ubuntu-latest + # Same image the pipelines publish from, so the pandoc contract tests check + # the pandoc that actually renders the wiki, not whatever a fresh runner has. + container: + image: pi5-16.local:3005/novoyuuparosk-wiki/novoyuuparosk-wiki-runner:4f134af + + steps: + - name: Checkout auto-wiki + env: + FAPAT: ${{ secrets.FAPAT }} + URL_TO_GITEA: ${{ vars.URL_TO_GITEA }} + REF: ${{ github.sha }} + run: | + git clone "http://mikkeli:${FAPAT}@${URL_TO_GITEA#http://}/novoyuuparosk-wiki/novoyuuparosk-auto-wiki" auto-wiki + git -C auto-wiki checkout "$REF" + + - name: Install test dependencies + run: pip install --no-cache-dir -r auto-wiki/requirements-dev.txt + + - name: Run tests + run: python -m pytest auto-wiki/tests -q diff --git a/lib/wiki.py b/lib/wiki.py index 1448bc4..0040678 100644 --- a/lib/wiki.py +++ b/lib/wiki.py @@ -4,6 +4,8 @@ import html import os import re import subprocess +import urllib.parse +from pathlib import Path from urllib.parse import urlparse import mwclient @@ -22,6 +24,38 @@ _BOLD_ITALIC_RE = re.compile(r"\*\*\*(.+?)\*\*\*") _BOLD_RE = re.compile(r"\*\*(.+?)\*\*") _ITALIC_RE = re.compile(r"\*(.+?)\*") +# Pandoc's mediawiki writer renders a *relative* Markdown link target as a +# wikilink carrying the raw path — [label](notes.md) becomes [[notes.md|label]] — +# and an absolute URL as [url label]. Both shapes are matched here so that a +# pandoc version which classifies relative targets the other way still resolves. +# The alternation is scanned left to right, so a [[...]] is consumed whole and +# the single-bracket branch never re-matches its interior. +_MD_LINK_RE = re.compile( + r"\[\[(?P[^\[\]|]+?)(?:\|(?P[^\[\]]*))?\]\]" + r"|\[(?P[^\[\]\s|]+)(?:[ \t]+(?P[^\[\]]*))?\]" +) + +# Inline Markdown link target, read from the *source* body before Pandoc runs: +# the "…](target)" tail of [label](target). Used for the link graph only. +_MD_SOURCE_LINK_RE = re.compile(r"\]\(\s*\s]+)>?") + +# Anything still naming a .md file in a link position after resolution means the +# renderer failed to understand it. That aborts the run; it never reaches the wiki. +_UNRESOLVED_LINK_RES = ( + ( + re.compile(r"\[\[[^\[\]]*\.md(?:#[^\[\]]*)?(?:\|[^\[\]]*)?\]\]", re.IGNORECASE), + "unresolved wikilink", + ), + ( + re.compile(r"\[[^\[\]\s|]*\.md[^\[\]]*\]", re.IGNORECASE), + "unresolved external-style link", + ), + ( + re.compile(r"\]\([^()\s]*\.md[^()]*\)", re.IGNORECASE), + "raw Markdown link (Markdown is not processed inside a ```columns fence)", + ), +) + def _md_emphasis_to_wikitext(text: str) -> str: text = _BOLD_ITALIC_RE.sub(r"'''''\1'''''", text) @@ -65,12 +99,100 @@ def expand_columns(wikitext: str) -> str: return _COLUMNS_BLOCK_RE.sub(render, wikitext) +def extract_md_link_targets(markdown_body: str) -> list[str]: + """Raw targets of inline Markdown links in an unrendered body. + + Used to build the link graph *before* Pandoc runs, so the publisher can work + out which pages a changed page points at (and which point back) without + rendering the whole repository. Targets are returned verbatim, including any + ``#fragment`` and percent-encoding; the caller resolves them. + """ + return _MD_SOURCE_LINK_RE.findall(markdown_body) + + +def assert_no_unresolved_md_links(wikitext: str, source_path: Path) -> None: + """Abort if any link still names a .md file after resolution. + + Guards against a link shape the resolver does not recognise: a pandoc version + that writes relative targets differently, or a Markdown link buried in a + verbatim ```columns fence. Without this, such a link would reach the wiki as + literal text, silently corrupting a published page. + """ + for pattern, kind in _UNRESOLVED_LINK_RES: + found = pattern.search(wikitext) + if found: + raise ValueError( + f"{source_path}: {kind} still references a .md file after " + f"resolution: {found.group(0)!r}" + ) + + +def resolve_md_links(wikitext: str, source_path: Path, title_index: dict[str, str]) -> str: + """Rewrite links to sibling ``.md`` files into wikilinks to their wiki pages. + + A related co-publication is nominated with an ordinary relative Markdown + link — ``see [dev notes](pulse_dev.md)`` — which stays a working link in + Gitea's blob view and in any Markdown preview. Here the ``.md`` target is + resolved against ``title_index`` (absolute path string → wiki page title) and + replaced with ``[[Wiki page title|label]]``. + + ``title_index`` must cover every publishable file in the repository, not only + the batch being published, or the same link resolves differently depending on + which files a push happened to touch. + + A ``.md`` target that cannot be resolved raises ``ValueError``: either the + file does not exist, or it exists but is not published. Both are authoring + mistakes, and both fail the run rather than silently degrade the page. Link + targets that do not name a ``.md`` file are left exactly as Pandoc wrote them. + """ + + def resolve(match: re.Match) -> str: + groups = match.groupdict() + if groups["wiki_target"] is not None: + target, label = groups["wiki_target"], groups["wiki_label"] + else: + target, label = groups["ext_target"], groups["ext_label"] + + raw_path, _, fragment = target.partition("#") + rel_path = urllib.parse.unquote(raw_path) + if not rel_path.lower().endswith(".md"): + return match.group(0) + + resolved = (source_path.parent / rel_path).resolve() + title = title_index.get(str(resolved)) + if title is None: + reason = ( + "file exists but is not published (no 'wiki.publish: true', " + "or it lives under wip/)" + if resolved.exists() + else "file does not exist" + ) + raise ValueError( + f"{source_path}: link to '{target}' cannot be resolved - {reason}" + ) + + anchor = f"#{fragment}" if fragment else "" + # A label that repeats the path (pandoc's form for an unlabelled link) or + # repeats the page title needs no pipe: [[Title]] already displays it. + if label is None or label.strip() in (target, title): + return f"[[{title}{anchor}]]" + return f"[[{title}{anchor}|{label}]]" + + resolved = _MD_LINK_RE.sub(resolve, wikitext) + assert_no_unresolved_md_links(resolved, source_path) + return resolved + + def markdown_to_wikitext(body: str) -> str: + # encoding is pinned rather than left to text=True, which would decode + # pandoc's output with the ambient locale encoding. The lyrics are Japanese + # and Chinese; on any non-UTF-8 locale that decode dies inside subprocess's + # reader thread, stdout comes back None, and the real error is lost. result = subprocess.run( ["pandoc", "-f", "markdown", "-t", "mediawiki"], input=body, capture_output=True, - text=True, + encoding="utf-8", ) if result.returncode != 0: raise RuntimeError(f"pandoc failed: {result.stderr.strip()}") diff --git a/pipelines/songs/README.md b/pipelines/songs/README.md index e688f55..1ffea66 100644 --- a/pipelines/songs/README.md +++ b/pipelines/songs/README.md @@ -15,7 +15,8 @@ Implemented: - Auto-generated banner (`{{Auto-generated|source=...|commit=}}`) - Album category injection (`[[Category:Songs]]` + `[[Category:Album:]]` when `album` is set); additional categories via `categories` list - LRC file upload (idempotent SHA1 check) + footer link -- Inter-page sibling placeholder resolution (`[[TAG]]` → `[[wiki page title|label]]`) +- Inter-page link resolution: a relative Markdown link (`[label](notes.md)`) becomes `[[wiki page title|label]]`, resolved against a repo-wide index so it never depends on which files a push touched +- Repo-wide page-title collision detection, and render-before-write, both ahead of any wiki contact - MediaWiki bot API write with idempotency — no-op if wiki content matches generated output - Files without `wiki.publish: true` silently skipped - Pages with existing non-auto-generated content skipped (logged to stderr) — delete the page or add the banner to hand ownership to the bot @@ -65,7 +66,12 @@ See the `Dockerfile` at the repo root and `requirements.txt` in this directory. ## Modes -The pipeline always runs `--all`: every file with `wiki.publish: true` (not under `wip/`) is checked against the current wiki content. Writes only happen when content has changed (commit SHA in the banner drives idempotency). This is appropriate for the current repo size. +**Reading is always repo-wide; writing is scoped to the push.** Every run scans every file with `wiki.publish: true` (not under `wip/`), because link resolution and title-collision detection have to see the whole tree. What gets *written* is the push diff, plus one hop in each link direction: any page a changed page links to, and any page that links to a changed page. + +- `--all` — publish every publishable file. Used on the first push and on `workflow_dispatch`. +- `--files ` — publish these files and their one-hop link neighbours. Used for an ordinary push. + +Writes only happen when content has changed (commit SHA in the banner drives idempotency), so a page pulled in by the link closure is normally a no-op API read. ## One-off wiki setup @@ -82,11 +88,16 @@ The pipeline always runs `--all`: every file with `wiki.publish: true` (not unde | Page template | Designed from scratch (no existing wiki pages to mirror) | 2026-06-09 | | Album categories | Auto-injected `[[Category:Songs]]` + `[[Category:Album:]]` when `album` is set; category pages are user-written and not overwritten | 2026-06-09 | | Additional categories | Optional `categories` list in frontmatter; each entry → `[[Category:]]` | 2026-06-10 | -| `album` optional | Companion/sibling pages don't need an album; omitting the field emits no album category | 2026-06-10 | -| Sibling resolution | Parent declares siblings as a bill of materials (`path` + `tag` + optional `label`); `[[TAG]]` substituted post-Pandoc | 2026-06-10 | +| `album` optional | Companion pages don't need an album; omitting the field emits no album category | 2026-06-10 | +| ~~Sibling resolution~~ | ~~Parent declares siblings as a bill of materials (`path` + `tag` + optional `label`); `[[TAG]]` substituted post-Pandoc~~ — **retired 2026-09-09**, see below | 2026-06-10 | +| Inter-page links | Plain relative Markdown link (`[label](notes.md)`); target's own `title` frontmatter is the page name. No frontmatter declaration, no macro namespace, and the link still works in Gitea's blob view. Replaces the `wiki.siblings` bill of materials, which substituted a bare tag by unanchored string replace — no word boundary, no collision check, and it degraded a link to plain text whenever the target fell outside the batch | 2026-09-09 | +| Unresolvable link | Hard error, never graceful degradation. A link to a missing or unpublished file fails the run; linking to a file does not publish it | 2026-09-09 | +| Render before write | All pages in a run are rendered before the first is written, so a broken page leaves the wiki untouched rather than half-updated | 2026-09-09 | | Source-repo branch convention | `autowiki/` short-lived branches in `ncmr-songs` | 2026-06-09 | | Idempotency | Commit SHA in banner (`commit=`), not timestamp — stable across identical re-runs | 2026-06-09 | -| Publish mode | Always `--all`; no diff detection — small repo, simpler than fragile git-diff gating | 2026-06-09 | +| Publish mode | Scan repo-wide, write the push diff plus one-hop link neighbours. Publishing is not instant, so the write set is bounded; resolution never is, which is what makes bounding the write set safe | 2026-09-09 | +| Title collisions | Detected across the whole repository, before any wiki contact — not only among the files one push happened to touch | 2026-09-09 | +| Tests | `tests/` run by `.gitea/workflows/test.yml` in the publish runner image, so the pandoc-output contract the link resolver depends on is pinned by a test rather than by prose | 2026-09-09 | | Manual-page protection | Bot skips pages without the `{{Auto-generated` banner to avoid overwriting hand-written content | 2026-06-09 | | `workflow_call` abandoned | Cross-repo `workflow_call` fails — run token scoped to triggering repo; cannot clone private callee | 2026-06-09 | | Side-by-side columns | Authored as a ` ```columns ` fenced block (separator `===`); Pandoc emits `
`, expanded post-Pandoc into a flex `
` of `` columns. No wiki template or extension — all Pi-side before the API write. See [SCHEMA.md](SCHEMA.md#columns-side-by-side-shorthand) | 2026-06-14 | diff --git a/pipelines/songs/SCHEMA.md b/pipelines/songs/SCHEMA.md index 3b2da86..e3debbf 100644 --- a/pipelines/songs/SCHEMA.md +++ b/pipelines/songs/SCHEMA.md @@ -2,7 +2,7 @@ Source-file contract for the songs pipeline. Files in `novoyuuparosk-wiki/ncmr-songs` must follow this contract to be picked up by the auto-publisher. -The schema lives entirely in the YAML frontmatter block at the top of each `.md` file. The body below the frontmatter is plain markdown, rendered to MediaWiki wikitext by Pandoc — no special body conventions are enforced beyond the standard markdown grammar, except that `[[TAG]]` placeholders declared via `wiki.siblings` are substituted with resolved wikilinks after rendering. +The schema lives entirely in the YAML frontmatter block at the top of each `.md` file. The body below the frontmatter is plain markdown, rendered to MediaWiki wikitext by Pandoc — no special body conventions are enforced beyond the standard markdown grammar. There is no macro syntax and nothing to declare: a link to another song page is an ordinary relative Markdown link, resolved to a wikilink at publish time. ## File layout @@ -12,20 +12,16 @@ title: pulse under latent semantic envelopes album: Shirakaba Express Service wiki: publish: true - siblings: - - path: pulse_dev.md - tag: PULSE_DEV - label: development notes release_date: 2026-06-18 lrc: pulse.lrc --- # pulse under latent semantic envelopes -... body markdown, PULSE_DEV used inline ... +... body markdown; a related page is nominated as [development notes](pulse_dev.md) ... ``` -The renderer parses the frontmatter, validates required fields, then hands the body (with the first `# Heading` line removed) to Pandoc, then applies sibling substitutions on the wikitext output. +The renderer parses the frontmatter, validates required fields, hands the body (with the first `# Heading` line removed) to Pandoc, then resolves `.md` links in the wikitext output to the wiki page titles of their targets. ## Fields @@ -81,54 +77,54 @@ Example: in `ses/pulse.md`, `lrc: pulse.lrc` declares that `ses/pulse.lrc` is th If the LRC file is missing or not valid UTF-8, the upload is skipped with a warning — the page still publishes. -#### `wiki.siblings` (list of sibling declarations) - -Bill of materials for inter-page wikilinks. Each entry declares a sibling `.md` file and a tag that can be used as a `[[TAG]]` placeholder anywhere in the body. - -```yaml -wiki: - siblings: - - path: pulse_dev.md # relative to this file's directory - tag: PULSE_DEV # placeholder used in body as [[PULSE_DEV]] - label: development notes # optional display text -``` - -**Fields per entry:** - -| Field | Required for resolution | Purpose | -|---|---|---| -| `path` | yes | Relative path to the sibling `.md` file | -| `tag` | yes | Placeholder identifier; pipeline does a case-sensitive string replace of `TAG` in the rendered wikitext | -| `label` | no | Display text for the resolved link | - -An entry missing either `path` or `tag` is silently skipped — no substitution is attempted for that entry. - -**Resolution:** - -The pipeline resolves each declared sibling's `path` to its wiki page title by looking it up in the set of all publishable files processed in the same run. Substitution is a case-sensitive string replace of the bare tag. Substitution rules: - -| Condition | `TAG` becomes | -|---|---| -| `path` resolves to a published page, `label` present | `[[wiki page title\|label]]` | -| `path` resolves to a published page, no `label` | `[[wiki page title]]` | -| `path` not in published set (file exists but not published), `label` present | `label` (plain text) | -| `path` not in published set, no `label` | `TAG` (plain text) | - -Declared siblings whose file doesn't exist on disk are a **validation error**. Siblings that exist but aren't published (no `wiki.publish: true`) degrade gracefully per the table above. - -The siblings list is a bill of materials — declaring a sibling has no effect unless its `[[TAG]]` appears in the body. Unused entries are inert. - -**`--files` mode**: sibling resolution is limited to the files in that batch. Siblings pointing to files outside the batch degrade to plain text. This is acceptable since the pipeline always runs `--all` in the workflow. - ## Body Plain markdown. The renderer applies these transformations before and after Pandoc: 1. **Pre-Pandoc**: strip the first-line `# Heading` if present. 2. **Post-Pandoc**: expand `columns` fenced blocks into side-by-side wikitext (see below). -3. **Post-Pandoc**: substitute `TAG` placeholders declared in `wiki.siblings`. +3. **Post-Pandoc**: resolve links to sibling `.md` files into wikilinks (see below). -Do not embed raw wikitext-specific syntax (`{{Template}}`, raw `[[Wikilink]]` not declared via `siblings`, etc.) in the body unless you intend the literal output. Use markdown idioms; the renderer adds the metadata-derived bits (banner, LRC link, categories, sibling resolution) around Pandoc's output. +Do not embed raw wikitext-specific syntax (`{{Template}}`, raw `[[Wikilink]]`, etc.) in the body unless you intend the literal output. Use markdown idioms; the renderer adds the metadata-derived bits (banner, LRC link, categories, link resolution) around Pandoc's output. + +### Nominating a related co-publication + +Link to it the ordinary way, with a relative Markdown link: + +```markdown +For more behind-the-scene shenanigans, see [dev gibberish](pulse_dev.md). +``` + +There is nothing to declare in the frontmatter. The link stays a working link in Gitea's blob view, in an editor preview, and anywhere else the raw file is read — and on the wiki it becomes a wikilink to the target's page: + +```wikitext +For more behind-the-scene shenanigans, see [[pulse under latent semantic envelopes - development notes|dev gibberish]]. +``` + +**How the target becomes a page name.** The path is resolved relative to the linking file's directory, and the target file's own `title` frontmatter field is used verbatim as the wikilink target — the same string the bot writes that page to. There is no second name format to keep in sync: rename a page by editing its `title`, and every link aimed at it follows on the next publish. MediaWiki handles the first-letter capitalisation and the space-to-underscore in URLs itself. + +| Written in the body | Rendered wikitext | +|---|---| +| `[dev gibberish](pulse_dev.md)` | `[[\|dev gibberish]]` | +| `[pulse_dev.md](pulse_dev.md)` (label same as path) | `[[]]` | +| `[notes](../ses/pulse_dev.md)` | `[[\|notes]]` | +| `[notes](pulse_dev.md#Section)` | `[[#Section\|notes]]` | +| `[site](https://example.com)` | `[https://example.com site]` (untouched) | + +**A `.md` link that cannot be resolved fails the run.** Both cases abort with a non-zero exit before anything is published: + +| Condition | Result | +|---|---| +| Target file does not exist | **error** — `file does not exist` | +| Target exists but is not published (no `wiki.publish: true`, or under `wip/`) | **error** — `file exists but is not published` | + +Linking to a file does **not** publish it. Publication is always the target's own explicit `wiki.publish: true`; nothing is published as a side effect of being linked. If you want the link, opt the target in. + +**Scope of a run.** A push publishes the files it changed, plus — one hop in each direction — any page a changed page links to, and any page that links to a changed page. The second is what keeps inbound links correct when a page's `title` moves. Regardless of how few pages a run publishes, the whole repository is always scanned first, so link resolution and title-collision checks see every publishable file. + +**Anchors** (`pulse_dev.md#Section`) are passed through verbatim. MediaWiki derives its own section anchors from heading text, so an anchor that doesn't match a heading on the target page lands the reader at the top of it rather than at the section. Nothing validates this. + +**Links inside a `columns` fence do not work.** The fence is verbatim by design (see below); Pandoc does not process Markdown in it, so a link written there would reach the wiki as literal text. The renderer detects this and fails the run rather than publishing it. Put the link in ordinary body prose. ### Columns (side-by-side) shorthand @@ -157,29 +153,43 @@ The renderer turns this into a flexbox row of `` columns (one per `===`-de ## Renderer behaviour -For each `.md` with `wiki.publish: true` and not under `wip/`: +A run happens in three phases, in this order, and the wiki is not contacted until the third. Anything wrong in phase 1 or 2 aborts with a non-zero exit and the wiki untouched — never half-updated. -1. Parse and validate frontmatter. -2. Strip the leading `# Heading` from the body if present. -3. Pipe the body through `pandoc -f markdown -t mediawiki`, then expand any `columns` fenced blocks into side-by-side `` columns. -4. Substitute `TAG` placeholders with resolved `[[Page|Display]]` wikilinks (or fallback text). -5. Prepend the auto-generated banner: `{{Auto-generated|source=|commit=}}`. -6. If `lrc` is declared: upload the LRC file if its SHA1 has changed; append a `[[Media:…]]` link. -7. Append category tags: `[[Category:Songs]]` and `[[Category:Album:]]` if `album` is set; `[[Category:]]` for each entry in `categories`. -8. Read the current wiki page content via the MediaWiki API; if identical to the generated output, skip the write (idempotency — keeps the wiki history clean). -9. If the page exists and does not start with `{{Auto-generated`, skip with a warning (manual page protection). -10. Otherwise, write the page with an edit summary referencing the source commit. +**Phase 1 — scan and validate the whole repository.** For every `.md` outside `wip/` (and excluding `README.md`): + +1. Parse the frontmatter; skip the file silently unless `wiki.publish: true`. +2. Validate the fields (see *Validation errors* below). +3. Register `title` and fail on a collision with any other publishable file, anywhere in the tree. +4. Build the path → page-title index every `.md` link resolves through. + +**Phase 2 — render every page that this run will publish.** For each selected file: + +1. Strip the leading `# Heading` from the body if present. +2. Pipe the body through `pandoc -f markdown -t mediawiki`, then expand any `columns` fenced blocks into side-by-side `` columns. +3. Resolve `.md` links to `[[Page title|label]]` wikilinks; fail on any that cannot be resolved, and on any link shape that survived rendering unresolved. +4. Prepend the auto-generated banner: `{{Auto-generated|source=|commit=}}`. +5. If `lrc` is declared, append a `[[Media:…]]` link. +6. Append category tags: `[[Category:Songs]]` and `[[Category:Album:]]` if `album` is set; `[[Category:]]` for each entry in `categories`. + +**Phase 3 — write.** For each rendered page: + +1. If `lrc` is declared: upload the LRC file if its SHA1 has changed. +2. Read the current wiki page content via the MediaWiki API; if identical to the generated output, skip the write (idempotency — keeps the wiki history clean). +3. If the page exists and does not start with `{{Auto-generated`, skip with a warning (manual page protection). +4. Otherwise, write the page with an edit summary referencing the source commit. ## Implemented scope - Frontmatter parsing and validation +- Repo-wide page-title collision detection, before any wiki write - Pandoc-based markdown → wikitext body rendering - First-line h1 stripping - Banner template injection - LRC file upload and footer link -- Sibling placeholder resolution (`[[TAG]]` → `[[wiki page title|label]]`) +- Relative `.md` link resolution (`[label](notes.md)` → `[[wiki page title|label]]`), repo-wide and batch-independent - Album category injection (`[[Category:Songs]]` + `[[Category:Album:]]`) when `album` is set - Additional categories via `categories` list +- Render-before-write: a broken page fails the run instead of half-updating the wiki - Idempotent writes (no-op skip when content matches) - Manual-page protection (skip pages without auto-gen banner) - Files without `wiki.publish: true` silently skipped @@ -199,16 +209,18 @@ title: pulse under latent semantic envelopes album: Shirakaba Express Service wiki: publish: true - siblings: - - path: pulse_dev.md - tag: PULSE_DEV - label: development notes release_date: 2026-06-18 lrc: pulse.lrc --- ``` -### A companion/sibling document (no album) +Its body nominates the companion page with a plain relative link: + +```markdown +For more behind-the-scene shenanigans, see [dev gibberish](pulse_dev.md). +``` + +### A companion document (no album) ```yaml --- @@ -249,8 +261,11 @@ Files without a `wiki.publish: true` frontmatter field are silently skipped — - `wiki.publish` present but not a boolean - `release_date` (if present) does not parse as YYYY-MM-DD - `categories` (if present) is not a list -- `wiki.siblings` entry that has both `path` and `tag` declared, but the `path` file does not exist on disk -- `wiki.siblings` entry that is not a mapping -- Two publishable source files declare the same `title` (page-name collision) +- Two publishable source files declare the same `title` (page-name collision) — checked across the whole repository, not just the files a push touched +- A `.md` link whose target file does not exist +- A `.md` link whose target exists but is not published +- A `.md` link that survives rendering unresolved — a link shape the renderer does not recognise, or a Markdown link written inside a `columns` fence -Validation runs before any wiki API calls. A failed validation aborts the run with a non-zero exit and leaves the wiki untouched. +Validation and rendering both run before any wiki write. A failure at either stage aborts the run with a non-zero exit and leaves the wiki untouched, including when only some of the run's pages are broken. + +Frontmatter keys the schema does not define are ignored, so a leftover `wiki.siblings` block from the retired macro system is inert rather than an error. Delete them; they do nothing. diff --git a/pipelines/songs/publish.py b/pipelines/songs/publish.py index 8272be2..ff81f35 100644 --- a/pipelines/songs/publish.py +++ b/pipelines/songs/publish.py @@ -6,6 +6,7 @@ import hashlib import io import os import sys +import urllib.parse from datetime import datetime from pathlib import Path @@ -13,7 +14,14 @@ import frontmatter import mwclient sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext, strip_first_h1 +from lib.wiki import ( + AUTO_BANNER_PREFIX, + connect_wiki, + extract_md_link_targets, + markdown_to_wikitext, + resolve_md_links, + strip_first_h1, +) def validate(fm: dict, path: Path) -> None: @@ -35,47 +43,58 @@ def validate(fm: dict, path: Path) -> None: if "categories" in fm and not isinstance(fm["categories"], list): errors.append("'categories' must be a list") - for i, sib in enumerate(fm.get("wiki", {}).get("siblings") or []): - if not isinstance(sib, dict): - errors.append(f"wiki.siblings[{i}] must be a mapping") - continue - sib_path = sib.get("path") - sib_tag = sib.get("tag") - if sib_path and sib_tag: - if not (path.parent / str(sib_path)).resolve().exists(): - errors.append(f"wiki.siblings[{i}]: path '{sib_path}' not found") - if errors: raise ValueError(f"{path}: " + "; ".join(errors)) -def build_sibling_map(posts: dict) -> dict[str, str]: +def build_title_index(posts: dict) -> dict[str, str]: """Map abs_path_str → wiki_title for all published posts.""" return {str(path): str(post.metadata["title"]) for path, post in posts.items()} -def apply_sibling_substitutions(wikitext: str, this_path: Path, fm_wiki: dict, sibling_map: dict[str, str]) -> str: - """Replace [[TAG]] placeholders using the declared sibling bill of materials.""" - for sib in (fm_wiki.get("siblings") or []): - if not isinstance(sib, dict): - continue - tag = sib.get("tag") - path = sib.get("path") - if not tag or not path: - continue - label = (sib.get("label") or "").strip() - abs_sib = str((this_path.parent / str(path)).resolve()) - title = sibling_map.get(abs_sib) - if title and label: - replacement = f"[[{title}|{label}]]" - elif title: - replacement = f"[[{title}]]" - elif label: - replacement = label - else: - replacement = tag - wikitext = wikitext.replace(tag, replacement) - return wikitext +def build_link_graph(posts: dict) -> dict[Path, set[Path]]: + """Map each publishable file to the publishable .md files it links to. + + Read from the raw Markdown, so it costs no Pandoc runs. Targets outside the + publishable set are dropped here; `resolve_md_links` is what refuses them + loudly at render time. + """ + graph: dict[Path, set[Path]] = {} + for path, post in posts.items(): + targets = set() + for raw in extract_md_link_targets(post.content): + rel_path = urllib.parse.unquote(raw.partition("#")[0]) + if not rel_path.lower().endswith(".md"): + continue + resolved = (path.parent / rel_path).resolve() + if resolved in posts: + targets.add(resolved) + graph[path] = targets + return graph + + +def select_publish_set(posts: dict, seeds: set[Path]) -> dict[Path, object]: + """The changed files, plus the pages their links touch in either direction. + + Publishing is not instant, so only what a push actually affects is written. + Two extra hops are unavoidable, both one level deep: + + - **outbound**: a page the changed page links to, so the wikilink cannot + point at a page that was never published (a red link); + - **inbound**: a page linking *to* a changed page, because if the changed + page's `title` moved, every link aimed at it is now stale. + + Ordering follows `posts` so a run's output is stable regardless of which + files the diff reported. + """ + graph = build_link_graph(posts) + selected = set(seeds) + for path, targets in graph.items(): + if path in seeds: + selected |= targets + elif targets & seeds: + selected.add(path) + return {path: post for path, post in posts.items() if path in selected} def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: str, lrc_filename: str | None = None) -> str: @@ -84,7 +103,7 @@ def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: st cat_parts = [] if fm.get("album"): - cat_parts.append(f"[[Category:Songs]]") + cat_parts.append("[[Category:Songs]]") cat_parts.append(f"[[Category:Album:{fm['album']}]]") for cat in (fm.get("categories") or []): cat_parts.append(f"[[Category:{cat}]]") @@ -121,25 +140,33 @@ def upload_lrc(lrc_path: Path, title: str, site: mwclient.Site) -> bool: return True -def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, source_ref: str, gitea_repo_url: str, sibling_map: dict[str, str]) -> str: - """Returns 'published', 'noop', or 'skipped-manual'.""" +def render_page(abs_path: Path, source_dir: Path, post, source_ref: str, gitea_repo_url: str, title_index: dict[str, str]) -> str: + """Render one file to finished wikitext. Pure: touches no wiki, writes nothing. + + Kept separate from the write so that every page in a run is rendered before + the first one is published. A bad link or a broken column fence then fails + the run with the wiki untouched, instead of half-updating it. + """ rel = abs_path.relative_to(source_dir) fm = post.metadata title = str(fm["title"]) body_wikitext = markdown_to_wikitext(strip_first_h1(post.content)) - body_wikitext = apply_sibling_substitutions(body_wikitext, abs_path, fm.get("wiki", {}), sibling_map) + body_wikitext = resolve_md_links(body_wikitext, abs_path, title_index) source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" - lrc_filename = None - lrc_rel = fm.get("lrc") - if lrc_rel: - lrc_uploaded = upload_lrc(abs_path.parent / str(lrc_rel), title, site) - lrc_filename = f"{title}.lrc" - if lrc_uploaded: - print(f" [lrc-uploaded] {lrc_filename}") + lrc_filename = f"{title}.lrc" if fm.get("lrc") else None + return build_wikitext(fm, body_wikitext, source_url, source_ref, lrc_filename) - page_content = build_wikitext(fm, body_wikitext, source_url, source_ref, lrc_filename) + +def publish_one(abs_path: Path, post, page_content: str, site: mwclient.Site, source_ref: str) -> str: + """Write one pre-rendered page. Returns 'published', 'noop', or 'skipped-manual'.""" + fm = post.metadata + title = str(fm["title"]) + + lrc_rel = fm.get("lrc") + if lrc_rel and upload_lrc(abs_path.parent / str(lrc_rel), title, site): + print(f" [lrc-uploaded] {title}.lrc") page = site.pages[title] existing = page.text() @@ -155,13 +182,29 @@ def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, sou return "published" -def load_and_validate(files: list[Path], source_dir: Path) -> dict[Path, object]: - posts = {} +def discover_publishable(source_dir: Path) -> dict[Path, object]: + """Every publishable .md in the repository, validated, keyed by absolute path. + + The whole tree is scanned on every run, never a subset of it, however few + pages the run then publishes. Two things depend on seeing all of it: + + - **link resolution**: a link to a sibling .md resolves through this set, so + a partial scan would resolve the same link differently depending on which + files a push happened to touch, and silently drop it when the target fell + outside the subset; + - **title collisions**: two files claiming one wiki page name must fail the + run wherever they sit in the tree, not only when both land in one push. + + Files under wip/ and README.md are never published. Everything else needs an + explicit `wiki.publish: true`; a file is never published as a side effect of + something else linking to it. + """ + posts: dict[Path, object] = {} titles: dict[str, Path] = {} - for path in files: - if not path.exists(): - print(f" warn: {path} not found, skipping", file=sys.stderr) + for path in sorted(source_dir.rglob("*.md")): + rel = path.relative_to(source_dir) + if "wip" in rel.parts or path.name == "README.md": continue post = frontmatter.load(str(path)) @@ -175,10 +218,11 @@ def load_and_validate(files: list[Path], source_dir: Path) -> dict[Path, object] title = str(post.metadata["title"]) if title in titles: raise ValueError( - f"Duplicate wiki title '{title}': {path} and {titles[title]}" + f"Duplicate wiki title '{title}': {rel} and " + f"{titles[title].relative_to(source_dir)}" ) titles[title] = path - posts[path] = post + posts[path.resolve()] = post return posts @@ -186,45 +230,56 @@ def load_and_validate(files: list[Path], source_dir: Path) -> dict[Path, object] def main() -> None: parser = argparse.ArgumentParser(description="Publish song .md files to MediaWiki.") parser.add_argument("--source-dir", required=True, type=Path) - parser.add_argument("--files", nargs="*", default=[], help="Relative paths within source-dir") - parser.add_argument("--all", action="store_true", help="Publish all publishable files") + parser.add_argument( + "--files", + nargs="*", + default=[], + help="Changed files (relative to source-dir) to publish; the push diff", + ) + parser.add_argument("--all", action="store_true", help="Publish every publishable file") args = parser.parse_args() source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + # The repository is always scanned in full, whatever gets published: link + # resolution and title-collision checks must see every publishable file, or + # the same link resolves differently depending on which files a push touched. + posts = discover_publishable(source_dir) + if not posts: + print("No publishable files found.") + return + + title_index = build_title_index(posts) + print(f"Scanned repository: {len(posts)} publishable file(s).") + if args.all: - candidates = list(source_dir.rglob("*.md")) + selected = posts elif args.files: - candidates = [source_dir / f for f in args.files] + seeds = {(source_dir / f).resolve() for f in args.files} & posts.keys() + if not seeds: + print("No changed file is publishable. Nothing to publish.") + return + selected = select_publish_set(posts, seeds) else: print("Nothing to do: pass --files or --all.") return - files = [ - p for p in candidates - if p.exists() - and p.suffix == ".md" - and "wip" not in p.relative_to(source_dir).parts - and p.name != "README.md" - ] - - if not files: - print("No publishable candidates after filtering.") - return - - print(f"Validating {len(files)} file(s)...") - posts = load_and_validate(files, source_dir) - sibling_map = build_sibling_map(posts) - print("Validation passed.") + # Render every page before writing any of them, so a bad link fails the run + # with the wiki untouched rather than half-updated. + rendered = { + abs_path: render_page(abs_path, source_dir, post, source_ref, gitea_repo_url, title_index) + for abs_path, post in selected.items() + } + print(f"Rendered {len(rendered)} page(s). Publishing.") site = connect_wiki() counts = {"published": 0, "noop": 0, "skipped-manual": 0} - for abs_path, post in posts.items(): + for abs_path, post in selected.items(): rel = abs_path.relative_to(source_dir) - outcome = publish_one(abs_path, source_dir, post, site, source_ref, gitea_repo_url, sibling_map) + outcome = publish_one(abs_path, post, rendered[abs_path], site, source_ref) counts[outcome] += 1 if outcome == "skipped-manual": print(f" [skipped-manual] {rel} ← page exists without auto-gen banner; delete or add banner to hand over to bot", file=sys.stderr) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e144ed0 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +pytest>=8.0.0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6e01ace --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,42 @@ +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + + +def _load_pipeline(name: str): + """Import a pipeline's publish.py under a unique module name. + + The three pipelines each define a module called `publish`, so they cannot be + imported by plain name without colliding in sys.modules. + """ + spec = importlib.util.spec_from_file_location( + f"{name}_publish", REPO_ROOT / "pipelines" / name / "publish.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="session") +def songs(): + return _load_pipeline("songs") + + +@pytest.fixture +def repo(tmp_path): + """Build a source tree of .md files: write(relative_path, frontmatter, body).""" + + def write(rel: str, front: str = "", body: str = "") -> Path: + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"---\n{front}\n---\n\n{body}\n", encoding="utf-8") + return path + + write.root = tmp_path + return write diff --git a/tests/test_songs_publish.py b/tests/test_songs_publish.py new file mode 100644 index 0000000..d04b5d6 --- /dev/null +++ b/tests/test_songs_publish.py @@ -0,0 +1,169 @@ +"""Tests for the songs publisher: discovery, validation, and publish scoping. + +None of these touch a wiki. `discover_publishable` and `select_publish_set` are +pure filesystem reads, which is the point — every failure they catch must fail +the run before `connect_wiki()` is ever called. +""" + +import pytest + +PUBLISHED = "title: {title}\nwiki:\n publish: true" +UNPUBLISHED = "title: {title}\nwiki:\n publish: false" + + +class TestDiscoverPublishable: + def test_collects_only_opted_in_files(self, songs, repo): + repo("a.md", PUBLISHED.format(title="A")) + repo("b.md", UNPUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + assert [p.metadata["title"] for p in posts.values()] == ["A"] + + def test_skips_wip_directory_even_when_opted_in(self, songs, repo): + repo("a.md", PUBLISHED.format(title="A")) + repo("wip/draft.md", PUBLISHED.format(title="Draft")) + posts = songs.discover_publishable(repo.root) + assert [p.metadata["title"] for p in posts.values()] == ["A"] + + def test_skips_readme(self, songs, repo): + repo("README.md", PUBLISHED.format(title="Readme")) + assert songs.discover_publishable(repo.root) == {} + + def test_scans_nested_directories(self, songs, repo): + repo("ses/a.md", PUBLISHED.format(title="A")) + repo("kairo/deep/b.md", PUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + assert sorted(p.metadata["title"] for p in posts.values()) == ["A", "B"] + + def test_duplicate_titles_in_different_directories_fail(self, songs, repo): + repo("ses/a.md", PUBLISHED.format(title="Same page")) + repo("kairo/b.md", PUBLISHED.format(title="Same page")) + with pytest.raises(ValueError, match="Duplicate wiki title 'Same page'"): + songs.discover_publishable(repo.root) + + def test_duplicate_title_error_names_both_files(self, songs, repo): + repo("ses/a.md", PUBLISHED.format(title="Same page")) + repo("kairo/b.md", PUBLISHED.format(title="Same page")) + with pytest.raises(ValueError) as err: + songs.discover_publishable(repo.root) + assert "a.md" in str(err.value) and "b.md" in str(err.value) + + def test_a_title_colliding_with_an_unpublished_file_is_allowed(self, songs, repo): + repo("a.md", PUBLISHED.format(title="Same page")) + repo("b.md", UNPUBLISHED.format(title="Same page")) + assert len(songs.discover_publishable(repo.root)) == 1 + + def test_publish_as_a_string_is_rejected(self, songs, repo): + repo("a.md", 'title: A\nwiki:\n publish: "true"') + with pytest.raises(ValueError, match="must be a boolean"): + songs.discover_publishable(repo.root) + + def test_missing_title_is_rejected(self, songs, repo): + repo("a.md", "wiki:\n publish: true") + with pytest.raises(ValueError, match="missing or empty 'title'"): + songs.discover_publishable(repo.root) + + def test_bad_release_date_is_rejected(self, songs, repo): + repo("a.md", PUBLISHED.format(title="A") + "\nrelease_date: 14-06-2026") + with pytest.raises(ValueError, match="YYYY-MM-DD"): + songs.discover_publishable(repo.root) + + def test_categories_must_be_a_list(self, songs, repo): + repo("a.md", PUBLISHED.format(title="A") + "\ncategories: kairo") + with pytest.raises(ValueError, match="'categories' must be a list"): + songs.discover_publishable(repo.root) + + def test_a_leftover_siblings_key_is_ignored_not_an_error(self, songs, repo): + # The old macro declaration is inert; a stale one must not fail a run. + repo("a.md", PUBLISHED.format(title="A") + "\n siblings:\n - tag: X") + assert len(songs.discover_publishable(repo.root)) == 1 + + +class TestSelectPublishSet: + def test_publishes_only_the_changed_file(self, songs, repo): + a = repo("a.md", PUBLISHED.format(title="A")) + repo("b.md", PUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + selected = songs.select_publish_set(posts, {a.resolve()}) + assert [p.metadata["title"] for p in selected.values()] == ["A"] + + def test_arms_the_page_a_changed_page_links_to(self, songs, repo): + a = repo("a.md", PUBLISHED.format(title="A"), "see [notes](b.md)") + repo("b.md", PUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + selected = songs.select_publish_set(posts, {a.resolve()}) + assert sorted(p.metadata["title"] for p in selected.values()) == ["A", "B"] + + def test_arms_the_page_that_links_to_a_changed_page(self, songs, repo): + repo("a.md", PUBLISHED.format(title="A"), "see [notes](b.md)") + b = repo("b.md", PUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + selected = songs.select_publish_set(posts, {b.resolve()}) + assert sorted(p.metadata["title"] for p in selected.values()) == ["A", "B"] + + def test_unrelated_pages_stay_out(self, songs, repo): + a = repo("a.md", PUBLISHED.format(title="A"), "see [notes](b.md)") + repo("b.md", PUBLISHED.format(title="B")) + repo("c.md", PUBLISHED.format(title="C")) + posts = songs.discover_publishable(repo.root) + selected = songs.select_publish_set(posts, {a.resolve()}) + assert "C" not in [p.metadata["title"] for p in selected.values()] + + def test_links_across_directories_are_followed(self, songs, repo): + a = repo("ses/a.md", PUBLISHED.format(title="A"), "see [notes](../kairo/b.md)") + repo("kairo/b.md", PUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + selected = songs.select_publish_set(posts, {a.resolve()}) + assert sorted(p.metadata["title"] for p in selected.values()) == ["A", "B"] + + def test_a_link_to_an_unpublished_file_adds_nothing(self, songs, repo): + # resolve_md_links is what rejects this loudly; the graph just drops it. + a = repo("a.md", PUBLISHED.format(title="A"), "see [draft](b.md)") + repo("b.md", UNPUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + selected = songs.select_publish_set(posts, {a.resolve()}) + assert [p.metadata["title"] for p in selected.values()] == ["A"] + + def test_a_self_link_does_not_duplicate(self, songs, repo): + a = repo("a.md", PUBLISHED.format(title="A"), "see [self](a.md)") + posts = songs.discover_publishable(repo.root) + selected = songs.select_publish_set(posts, {a.resolve()}) + assert len(selected) == 1 + + +class TestBuildTitleIndex: + def test_maps_every_publishable_path_to_its_title(self, songs, repo): + a = repo("a.md", PUBLISHED.format(title="A")) + posts = songs.discover_publishable(repo.root) + assert songs.build_title_index(posts)[str(a.resolve())] == "A" + + def test_covers_the_whole_repo_not_just_a_changed_file(self, songs, repo): + repo("a.md", PUBLISHED.format(title="A")) + b = repo("nested/b.md", PUBLISHED.format(title="B")) + posts = songs.discover_publishable(repo.root) + assert str(b.resolve()) in songs.build_title_index(posts) + + +class TestBuildWikitext: + def test_album_injects_both_song_and_album_categories(self, songs): + out = songs.build_wikitext({"album": "kairo II"}, "body\n", "url", "sha") + assert "[[Category:Songs]]" in out + assert "[[Category:Album:kairo II]]" in out + + def test_no_album_injects_no_category(self, songs): + out = songs.build_wikitext({}, "body\n", "url", "sha") + assert "[[Category:" not in out + + def test_extra_categories_are_additive(self, songs): + out = songs.build_wikitext( + {"album": "kairo II", "categories": ["kairo"]}, "body\n", "url", "sha" + ) + assert "[[Category:kairo]]" in out + assert "[[Category:Album:kairo II]]" in out + + def test_banner_carries_source_and_commit(self, songs): + out = songs.build_wikitext({}, "body\n", "http://g/x", "abc123") + assert out.startswith("{{Auto-generated|source=http://g/x|commit=abc123}}") + + def test_lrc_footer_link_is_appended_when_named(self, songs): + out = songs.build_wikitext({}, "body\n", "url", "sha", "A.lrc") + assert "[[Media:A.lrc|Synced lyrics (.lrc)]]" in out diff --git a/tests/test_wiki.py b/tests/test_wiki.py new file mode 100644 index 0000000..37f5879 --- /dev/null +++ b/tests/test_wiki.py @@ -0,0 +1,216 @@ +"""Tests for the shared render helpers in lib/wiki.py. + +All pure string in, string out, except the pandoc contract test at the bottom, +which is the canary for a pandoc upgrade changing how relative links are written. +""" + +import shutil +import subprocess +from pathlib import Path + +import pytest + +from lib.wiki import ( + assert_no_unresolved_md_links, + expand_columns, + extract_md_link_targets, + markdown_to_wikitext, + resolve_md_links, + strip_first_h1, +) + +# Paths need not exist on disk: resolution is non-strict, and only the missing +# vs. unpublished distinction consults the filesystem (covered with tmp_path). +SONGS = Path(__file__).resolve().parent / "_paths" / "songs" +HERE = SONGS / "pulse.md" +INDEX = { + str((SONGS / "pulse.md").resolve()): "pulse under latent semantic envelopes", + str((SONGS / "pulse_dev.md").resolve()): ( + "pulse under latent semantic envelopes - development notes" + ), + str((SONGS / "two words.md").resolve()): "Two words", +} + + +def resolve(wikitext, source=HERE): + return resolve_md_links(wikitext, source, INDEX) + + +class TestStripFirstH1: + def test_removes_heading_and_following_blank_line(self): + assert strip_first_h1("# Title\n\nbody") == "body" + + def test_removes_only_the_first(self): + assert strip_first_h1("# One\n\n# Two\n") == "# Two\n" + + def test_leaves_body_without_h1_alone(self): + assert strip_first_h1("## Sub\n\nbody") == "## Sub\n\nbody" + + def test_finds_the_h1_even_when_a_deeper_heading_precedes_it(self): + assert strip_first_h1("### Deep\n\n# Title\n") == "### Deep\n" + + +class TestExpandColumns: + def test_splits_on_separator_and_wraps_each_in_poem(self): + out = expand_columns('
a\nb\n===\nc
') + assert out.count("") == 2 + assert "\na\nb\n" in out + assert "\nc\n" in out + assert out.startswith("
a\n===\nb\n===\nc
') + assert out.count("") == 3 + + def test_converts_markdown_emphasis(self): + out = expand_columns('
***a*** **b** *c*
') + assert "'''''a'''''" in out + assert "'''b'''" in out + assert "''c''" in out + + def test_unescapes_inline_html(self): + out = expand_columns('
<b>x</b>
') + assert "x" in out + + def test_leaves_wikitext_without_a_columns_block_untouched(self): + assert expand_columns("plain '''wikitext'''") == "plain '''wikitext'''" + + +class TestResolveMdLinks: + def test_labelled_link_becomes_piped_wikilink(self): + assert resolve("see [[pulse_dev.md|dev notes]] ok") == ( + "see [[pulse under latent semantic envelopes - development notes|dev notes]] ok" + ) + + def test_unlabelled_link_uses_the_page_title(self): + assert resolve("[[pulse_dev.md]]") == ( + "[[pulse under latent semantic envelopes - development notes]]" + ) + + def test_label_identical_to_target_drops_the_pipe(self): + assert resolve("[[pulse_dev.md|pulse_dev.md]]") == ( + "[[pulse under latent semantic envelopes - development notes]]" + ) + + def test_label_identical_to_the_page_title_drops_the_pipe(self): + assert resolve("[[pulse_dev.md|pulse under latent semantic envelopes - development notes]]") == ( + "[[pulse under latent semantic envelopes - development notes]]" + ) + + def test_fragment_is_carried_onto_the_title(self): + assert resolve("[[pulse_dev.md#Notes|x]]") == ( + "[[pulse under latent semantic envelopes - development notes#Notes|x]]" + ) + + def test_percent_encoding_in_target_is_decoded(self): + assert resolve("[[two%20words.md|x]]") == "[[Two words|x]]" + + def test_parent_directory_target_resolves(self): + assert resolve_md_links( + "[[../pulse_dev.md|x]]", SONGS / "sub" / "a.md", INDEX + ) == "[[pulse under latent semantic envelopes - development notes|x]]" + + def test_external_style_link_shape_also_resolves(self): + # Insurance against a pandoc version writing relative targets this way. + assert resolve("[pulse_dev.md dev notes]") == ( + "[[pulse under latent semantic envelopes - development notes|dev notes]]" + ) + + def test_absolute_url_is_left_alone(self): + assert resolve("[https://example.com/x label]") == "[https://example.com/x label]" + + def test_non_md_wikilink_is_left_alone(self): + assert resolve("[[Category:Songs]] [[Media:x.lrc|y]]") == ( + "[[Category:Songs]] [[Media:x.lrc|y]]" + ) + + def test_label_containing_a_pipe_is_preserved(self): + assert resolve("[[pulse_dev.md|a|b]]").endswith("|a|b]]") + + def test_several_links_in_one_body(self): + out = resolve("[[pulse_dev.md|a]] then [[two%20words.md|b]]") + assert "|a]]" in out and "[[Two words|b]]" in out + + def test_missing_file_is_an_error(self): + with pytest.raises(ValueError, match="does not exist"): + resolve("[[nope.md|x]]") + + def test_existing_but_unpublished_file_is_an_error(self, tmp_path): + (tmp_path / "draft.md").write_text("x", encoding="utf-8") + with pytest.raises(ValueError, match="not published"): + resolve_md_links("[[draft.md|x]]", tmp_path / "song.md", {}) + + def test_error_names_the_source_file(self): + with pytest.raises(ValueError, match=r"pulse\.md"): + resolve("[[nope.md|x]]") + + +class TestUnresolvedLinkGuard: + def test_accepts_output_with_no_md_reference(self): + assert_no_unresolved_md_links("[[A page|x]] and [http://e.com y]", HERE) + + def test_rejects_leftover_wikilink(self): + with pytest.raises(ValueError, match="unresolved wikilink"): + assert_no_unresolved_md_links("[[notes.md|x]]", HERE) + + def test_rejects_leftover_external_style_link(self): + with pytest.raises(ValueError, match="external-style"): + assert_no_unresolved_md_links("[notes.md x]", HERE) + + def test_rejects_raw_markdown_link_left_in_a_columns_fence(self): + with pytest.raises(ValueError, match="columns"): + assert_no_unresolved_md_links("\nsee [x](notes.md)\n", HERE) + + def test_prose_mentioning_a_filename_is_not_a_link(self): + assert_no_unresolved_md_links("edit pulse.md by hand", HERE) + + +class TestExtractMdLinkTargets: + def test_finds_inline_targets(self): + assert extract_md_link_targets("a [x](one.md) b [y](../two.md#f)") == [ + "one.md", + "../two.md#f", + ] + + def test_ignores_bodies_without_links(self): + assert extract_md_link_targets("no links here") == [] + + def test_finds_targets_inside_a_columns_fence(self): + # The fence cannot render them, but the graph must still see the edge. + assert extract_md_link_targets("```columns\n[x](one.md)\n```") == ["one.md"] + + +@pytest.mark.skipif(shutil.which("pandoc") is None, reason="pandoc not installed") +class TestPandocContract: + """Pins the pandoc output shape that resolve_md_links depends on. + + If a pandoc upgrade starts writing relative link targets differently, this + fails here instead of silently publishing a broken link. + """ + + def test_version(self): + out = subprocess.run( + ["pandoc", "--version"], capture_output=True, text=True, check=True + ).stdout + print(out.splitlines()[0]) + + def test_relative_link_survives_the_full_render(self): + wikitext = markdown_to_wikitext("see [dev notes](pulse_dev.md) here") + assert resolve(wikitext).strip() == ( + "see [[pulse under latent semantic envelopes - development notes|dev notes]] here" + ) + + def test_unlabelled_relative_link_survives_the_full_render(self): + wikitext = markdown_to_wikitext("[pulse_dev.md](pulse_dev.md)") + assert resolve(wikitext).strip() == ( + "[[pulse under latent semantic envelopes - development notes]]" + ) + + def test_absolute_url_survives_untouched(self): + wikitext = markdown_to_wikitext("[site](https://example.com/x)") + assert resolve(wikitext).strip() == "[https://example.com/x site]" + + def test_columns_fence_renders_to_side_by_side_poems(self): + wikitext = markdown_to_wikitext("```columns\n**A**\nl1\n===\n**B**\nl2\n```") + assert wikitext.count("") == 2 + assert "'''A'''" in wikitext