From e1810facd643c95dff6e1d6a99a93abe0747e823 Mon Sep 17 00:00:00 2001 From: Wan Ziyu Date: Wed, 9 Sep 2026 16:37:06 +0900 Subject: [PATCH] Split the public banner URL from the runner-internal clone URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every auto-published page, in all three pipelines, carried a source link to `localhost:3005`. The template was never at fault — it links `{{{source}}}` verbatim; the bad URL was baked into each page's wikitext by the publisher. Root cause was one variable serving two incompatible purposes. `URL_TO_GITEA` has to be an address the runner container can clone from, and since the runner uses host networking that is `http://localhost:3005`. All three workflows then composed `GITEA_REPO_URL` — the base of the public banner link, written into the wiki for anyone to read — out of that same loopback value. Adds a second org variable, `PUBLIC_URL_TO_GITEA`, for the public base (`https://git.novoyuuparosk.org`), and points the banner at it. Cloning keeps using `URL_TO_GITEA`. The two are now documented as deliberately separate, because collapsing them looks like tidying up. `lib/wiki.public_source_base()` validates the banner base rather than trusting it: unset, relative, loopback, mDNS (`*.local`) or private-range values raise with the variable to set. A misconfiguration that would publish an unreachable link now stops the run instead. Note the failure mode this creates on purpose: until `PUBLIC_URL_TO_GITEA` exists, publishing aborts rather than proceeding with a wrong URL. Also builds the source path with `rel.as_posix()`, so the separator in a URL does not depend on the OS the publisher runs on — it produced backslashes when the renderer was exercised on Windows. Prerequisites before this can publish: 1. Add the org variable `PUBLIC_URL_TO_GITEA` = https://git.novoyuuparosk.org 2. Flush the act_runner callee cache — `docker compose up -d --force-recreate act_runner`. Per the cross-cutting decisions log, edits to `publish-*.yml` keep executing the stale cached copy, and this change is in those files. Pages already published still carry the old URL in their stored wikitext. They are only corrected by re-publishing each pipeline in full. Co-Authored-By: Claude Opus 5 (1M context) --- .gitea/workflows/publish-ses.yml | 2 +- .gitea/workflows/publish-songs.yml | 2 +- .gitea/workflows/publish-tech.yml | 2 +- README.md | 8 +++- lib/wiki.py | 58 +++++++++++++++++++++++++ pipelines/ses/README.md | 5 ++- pipelines/ses/publish.py | 11 +++-- pipelines/songs/README.md | 5 ++- pipelines/songs/publish.py | 5 ++- pipelines/tech/README.md | 5 ++- pipelines/tech/publish.py | 11 +++-- tests/test_wiki.py | 70 ++++++++++++++++++++++++++++++ 12 files changed, 166 insertions(+), 18 deletions(-) diff --git a/.gitea/workflows/publish-ses.yml b/.gitea/workflows/publish-ses.yml index e59119b..5366f98 100644 --- a/.gitea/workflows/publish-ses.yml +++ b/.gitea/workflows/publish-ses.yml @@ -34,7 +34,7 @@ jobs: WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} SOURCE_REF: ${{ inputs.SOURCE_REF }} - GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/novoyuuparosk-wiki/ses-light-novel + GITEA_REPO_URL: ${{ vars.PUBLIC_URL_TO_GITEA }}/novoyuuparosk-wiki/ses-light-novel run: | python auto-wiki/pipelines/ses/publish.py \ --source-dir ses-light-novel \ diff --git a/.gitea/workflows/publish-songs.yml b/.gitea/workflows/publish-songs.yml index 29b46fe..9cc8a42 100644 --- a/.gitea/workflows/publish-songs.yml +++ b/.gitea/workflows/publish-songs.yml @@ -62,7 +62,7 @@ jobs: WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} SOURCE_REF: ${{ inputs.SOURCE_REF }} - GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/novoyuuparosk-wiki/ncmr-songs + GITEA_REPO_URL: ${{ vars.PUBLIC_URL_TO_GITEA }}/novoyuuparosk-wiki/ncmr-songs MODE: ${{ steps.diff.outputs.mode }} FILES: ${{ steps.diff.outputs.files }} run: | diff --git a/.gitea/workflows/publish-tech.yml b/.gitea/workflows/publish-tech.yml index 3d91f36..3c27565 100644 --- a/.gitea/workflows/publish-tech.yml +++ b/.gitea/workflows/publish-tech.yml @@ -34,7 +34,7 @@ jobs: WIKI_BOT_USER: ${{ secrets.WIKI_BOT_USER }} WIKI_BOT_PASSWORD: ${{ secrets.WIKI_BOT_PASSWORD }} SOURCE_REF: ${{ inputs.SOURCE_REF }} - GITEA_REPO_URL: ${{ vars.URL_TO_GITEA }}/novoyuuparosk-wiki/tech-blogs + GITEA_REPO_URL: ${{ vars.PUBLIC_URL_TO_GITEA }}/novoyuuparosk-wiki/tech-blogs run: | python auto-wiki/pipelines/tech/publish.py \ --source-dir tech-blogs \ diff --git a/README.md b/README.md index 0bfe6d5..584be18 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,10 @@ Secrets and variables are scoped to the `novoyuuparosk-wiki` org, inherited by a **Variables:** - `WIKI_BASE_URL` = `https://wiki.novoyuuparosk.org` - `WIKI_API_URL` = `https://wiki.novoyuuparosk.org/api.php` -- `URL_TO_GITEA` = Gitea instance base URL (e.g. `http://localhost:3005`). Named with `URL_TO_` prefix — Gitea blocks variable names starting with `GITEA_` or `GITHUB_`. +- `URL_TO_GITEA` = **runner-internal** Gitea base URL, used only for cloning (e.g. `http://localhost:3005` — the runner uses host networking). Named with `URL_TO_` prefix — Gitea blocks variable names starting with `GITEA_` or `GITHUB_`. +- `PUBLIC_URL_TO_GITEA` = **public** Gitea base URL (`https://git.novoyuuparosk.org`), used only for the source link in each published page's banner. + +These two must stay separate. `URL_TO_GITEA` is a loopback address, so composing the banner from it published every page — across all three pipelines — citing `localhost:3005`, which resolves for nobody reading the wiki. `lib/wiki.public_source_base()` now rejects a loopback, mDNS, or private-range value, so that misconfiguration fails the run instead of reaching a page. ## Branch naming @@ -98,6 +101,8 @@ Secrets and variables are scoped to the `novoyuuparosk-wiki` org, inherited by a | MediaWiki API path | `api.php` (classic action API) | 2026-06-09 | | Branch naming (this repo) | `automation/` for pipeline-development branches | 2026-06-09 | | Variable naming | `URL_TO_GITEA` not `GITEA_URL` — Gitea blocks `GITEA_`/`GITHUB_` prefixes | 2026-06-09 | +| **Clone URL ≠ banner URL** | Split into `URL_TO_GITEA` (runner-internal, for cloning) and `PUBLIC_URL_TO_GITEA` (public, for the page banner). One variable served both, and since host networking forces the clone address to `localhost:3005`, every auto-published page in every pipeline cited a link no reader could open. The banner value is validated by `public_source_base()` — loopback, mDNS and private-range addresses abort the run. Pages published before this carry the bad URL in their wikitext and need a full re-publish per pipeline to clear | 2026-09-09 | +| Source URL separators | `rel.as_posix()` in the banner URL, so the path separator never depends on the OS the publisher happens to run on | 2026-09-09 | | Columns shorthand | Side-by-side columns authored as a ` ```columns ` fenced block, expanded post-Pandoc in shared `lib/wiki.py` (universal across pipelines). No wiki template or PHP extension — runs Pi-side before the API call | 2026-06-14 | Per-pipeline decisions live in each pipeline's README. @@ -113,6 +118,7 @@ Via the Gitea web UI logged in as `mikkeli` (an owner of `novoyuuparosk-wiki`): - [x] `WIKI_BASE_URL` - [x] `WIKI_API_URL` - [x] `URL_TO_GITEA` + - [ ] `PUBLIC_URL_TO_GITEA` = `https://git.novoyuuparosk.org` — **required**; the publishers abort without it With Pi access: diff --git a/lib/wiki.py b/lib/wiki.py index 0040678..8ab8645 100644 --- a/lib/wiki.py +++ b/lib/wiki.py @@ -1,6 +1,7 @@ """Shared utilities for novoyuuparosk-auto-wiki pipelines.""" import html +import ipaddress import os import re import subprocess @@ -199,6 +200,63 @@ def markdown_to_wikitext(body: str) -> str: return expand_columns(result.stdout) +_LOOPBACK_HOSTNAMES = frozenset({"localhost", "localhost.localdomain", "ip6-localhost"}) + + +def public_source_base(env_var: str = "GITEA_REPO_URL") -> str: + """The public base URL for the source links baked into published pages. + + This URL ends up in the `{{Auto-generated}}` banner of every page, so it is + read by anyone on the internet — it is *not* the address the runner clones + from. Those two are different hosts here: the runner container uses host + networking and reaches Gitea on `localhost:3005`, while a reader needs the + public name. Composing the banner from the clone address published pages + citing `localhost:3005`, which is why this is validated rather than trusted. + + Rejects an address that cannot possibly work for a reader — unset, missing a + scheme or host, loopback, mDNS (`*.local`), or a private IP range — so the + misconfiguration fails the run instead of being written to the wiki. + """ + raw = os.environ.get(env_var, "").strip().rstrip("/") + if not raw: + raise RuntimeError( + f"{env_var} is unset. It must be the *public* Gitea base URL plus the " + f"repository path (e.g. https://git.example.org/org/repo); the workflow " + f"composes it from the PUBLIC_URL_TO_GITEA Gitea variable." + ) + + parsed = urlparse(raw) + if not parsed.scheme or not parsed.netloc: + raise RuntimeError( + f"{env_var} is not an absolute URL: {raw!r}. Expected scheme and host, " + f"e.g. https://git.example.org/org/repo." + ) + + host = (parsed.hostname or "").lower() + reason = None + if host in _LOOPBACK_HOSTNAMES: + reason = "a loopback hostname" + elif host.endswith(".local"): + reason = "an mDNS name, resolvable only on the local network" + else: + try: + address = ipaddress.ip_address(host) + except ValueError: + address = None + if address is not None and (address.is_loopback or address.is_private): + reason = "a loopback or private-range IP address" + + if reason: + raise RuntimeError( + f"{env_var} points at {reason}: {raw!r}. This URL is published in the " + f"page banner and must resolve for a reader off the network. Set the " + f"PUBLIC_URL_TO_GITEA Gitea variable to the public Gitea base URL — " + f"URL_TO_GITEA is the runner-internal clone address and is not usable here." + ) + + return raw + + def connect_wiki() -> mwclient.Site: api_url = os.environ["WIKI_API_URL"] parsed = urlparse(api_url) diff --git a/pipelines/ses/README.md b/pipelines/ses/README.md index 9fe7747..074e7b6 100644 --- a/pipelines/ses/README.md +++ b/pipelines/ses/README.md @@ -48,10 +48,11 @@ Triggers: | `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | | `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login | | `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | -| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Runner-internal Gitea base URL, for cloning only | +| `PUBLIC_URL_TO_GITEA` | variable | Gitea (cross-cutting) | Public Gitea base URL, for the banner source link only | | `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | | `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | -| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | +| `GITEA_REPO_URL` | env (set in workflow) | `PUBLIC_URL_TO_GITEA` + repo path | Source-file URL base for the banner link. Validated by `public_source_base()`: a loopback, mDNS or private-range host aborts the run rather than publishing an unreachable link | ## Dependencies diff --git a/pipelines/ses/publish.py b/pipelines/ses/publish.py index 7f5f2e1..ffef62a 100644 --- a/pipelines/ses/publish.py +++ b/pipelines/ses/publish.py @@ -10,7 +10,12 @@ import frontmatter import mwclient sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext +from lib.wiki import ( + AUTO_BANNER_PREFIX, + connect_wiki, + markdown_to_wikitext, + public_source_base, +) def validate(fm: dict, path: Path) -> None: @@ -59,7 +64,7 @@ def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, sou page_type = get_page_type(fm, abs_path, source_dir) body_wikitext = markdown_to_wikitext(post.content) - source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel.as_posix()}" page_content = build_wikitext(fm, body_wikitext, source_url, source_ref, page_type) page = site.pages[title] @@ -111,7 +116,7 @@ def main() -> None: source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") - gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + gitea_repo_url = public_source_base() if args.all: candidates = list(source_dir.rglob("*.md")) diff --git a/pipelines/songs/README.md b/pipelines/songs/README.md index 1ffea66..11de929 100644 --- a/pipelines/songs/README.md +++ b/pipelines/songs/README.md @@ -53,10 +53,11 @@ Triggers: | `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | | `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login (`Mikkeli@giteaBot`) | | `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | -| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Runner-internal Gitea base URL, for cloning only | +| `PUBLIC_URL_TO_GITEA` | variable | Gitea (cross-cutting) | Public Gitea base URL, for the banner source link only | | `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | | `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | -| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | +| `GITEA_REPO_URL` | env (set in workflow) | `PUBLIC_URL_TO_GITEA` + repo path | Source-file URL base for the banner link. Validated by `public_source_base()`: a loopback, mDNS or private-range host aborts the run rather than publishing an unreachable link | ## Dependencies diff --git a/pipelines/songs/publish.py b/pipelines/songs/publish.py index ff81f35..c533a99 100644 --- a/pipelines/songs/publish.py +++ b/pipelines/songs/publish.py @@ -19,6 +19,7 @@ from lib.wiki import ( connect_wiki, extract_md_link_targets, markdown_to_wikitext, + public_source_base, resolve_md_links, strip_first_h1, ) @@ -153,7 +154,7 @@ def render_page(abs_path: Path, source_dir: Path, post, source_ref: str, gitea_r body_wikitext = markdown_to_wikitext(strip_first_h1(post.content)) body_wikitext = resolve_md_links(body_wikitext, abs_path, title_index) - source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel.as_posix()}" lrc_filename = f"{title}.lrc" if fm.get("lrc") else None return build_wikitext(fm, body_wikitext, source_url, source_ref, lrc_filename) @@ -241,7 +242,7 @@ def main() -> None: source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") - gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + gitea_repo_url = public_source_base() # The repository is always scanned in full, whatever gets published: link # resolution and title-collision checks must see every publishable file, or diff --git a/pipelines/tech/README.md b/pipelines/tech/README.md index 4efefb4..4e51bde 100644 --- a/pipelines/tech/README.md +++ b/pipelines/tech/README.md @@ -42,10 +42,11 @@ Triggers: | `WIKI_BASE_URL` | variable | Gitea (cross-cutting) | Wiki base URL | | `WIKI_BOT_USER` | secret | Gitea (cross-cutting) | Bot login | | `WIKI_BOT_PASSWORD` | secret | Gitea (cross-cutting) | BotPasswords value | -| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Gitea instance base URL for cloning | +| `URL_TO_GITEA` | variable | Gitea (cross-cutting) | Runner-internal Gitea base URL, for cloning only | +| `PUBLIC_URL_TO_GITEA` | variable | Gitea (cross-cutting) | Public Gitea base URL, for the banner source link only | | `FAPAT` | secret | Gitea (cross-cutting) | Full-Access PAT for cloning private repos | | `SOURCE_REF` | env (set in workflow) | `github.sha` | Pushed commit SHA, used in banner and edit summary | -| `GITEA_REPO_URL` | env (set in workflow) | composed from vars | Source-file URL base for the banner link | +| `GITEA_REPO_URL` | env (set in workflow) | `PUBLIC_URL_TO_GITEA` + repo path | Source-file URL base for the banner link. Validated by `public_source_base()`: a loopback, mDNS or private-range host aborts the run rather than publishing an unreachable link | ## Dependencies diff --git a/pipelines/tech/publish.py b/pipelines/tech/publish.py index 6ba6091..e79b09e 100644 --- a/pipelines/tech/publish.py +++ b/pipelines/tech/publish.py @@ -11,7 +11,12 @@ import frontmatter import mwclient sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext +from lib.wiki import ( + AUTO_BANNER_PREFIX, + connect_wiki, + markdown_to_wikitext, + public_source_base, +) def validate(fm: dict, path: Path) -> None: @@ -58,7 +63,7 @@ def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, sou title = str(fm["title"]) body_wikitext = markdown_to_wikitext(post.content) - source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" + source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel.as_posix()}" page_content = build_wikitext(fm, body_wikitext, source_url, source_ref) page = site.pages[title] @@ -110,7 +115,7 @@ def main() -> None: source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") - gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") + gitea_repo_url = public_source_base() if args.all: candidates = list(source_dir.rglob("*.md")) diff --git a/tests/test_wiki.py b/tests/test_wiki.py index 37f5879..702af57 100644 --- a/tests/test_wiki.py +++ b/tests/test_wiki.py @@ -15,6 +15,7 @@ from lib.wiki import ( expand_columns, extract_md_link_targets, markdown_to_wikitext, + public_source_base, resolve_md_links, strip_first_h1, ) @@ -214,3 +215,72 @@ class TestPandocContract: wikitext = markdown_to_wikitext("```columns\n**A**\nl1\n===\n**B**\nl2\n```") assert wikitext.count("") == 2 assert "'''A'''" in wikitext + + +class TestPublicSourceBase: + """The banner URL is read by the public, so a runner-only address must fail. + + Every one of these rejections was a page published with a link to + localhost:3005, which is what the whole guard exists to prevent. + """ + + def test_accepts_a_public_https_url(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "https://git.novoyuuparosk.org/org/repo") + assert public_source_base() == "https://git.novoyuuparosk.org/org/repo" + + def test_accepts_a_public_host_with_a_port(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://git.novoyuuparosk.org:1551/org/repo") + assert public_source_base() == "http://git.novoyuuparosk.org:1551/org/repo" + + def test_strips_a_trailing_slash(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "https://git.example.org/org/repo/") + assert public_source_base() == "https://git.example.org/org/repo" + + def test_strips_surrounding_whitespace(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", " https://git.example.org/org/repo ") + assert public_source_base() == "https://git.example.org/org/repo" + + def test_unset_is_rejected(self, monkeypatch): + monkeypatch.delenv("GITEA_REPO_URL", raising=False) + with pytest.raises(RuntimeError, match="is unset"): + public_source_base() + + def test_empty_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "") + with pytest.raises(RuntimeError, match="is unset"): + public_source_base() + + def test_a_bare_path_is_rejected(self, monkeypatch): + # What an unset PUBLIC_URL_TO_GITEA leaves the workflow composing. + monkeypatch.setenv("GITEA_REPO_URL", "/novoyuuparosk-wiki/ncmr-songs") + with pytest.raises(RuntimeError, match="not an absolute URL"): + public_source_base() + + def test_localhost_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://localhost:3005/org/repo") + with pytest.raises(RuntimeError, match="loopback hostname"): + public_source_base() + + def test_loopback_ip_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://127.0.0.1:3005/org/repo") + with pytest.raises(RuntimeError, match="loopback or private-range"): + public_source_base() + + def test_private_range_ip_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://192.168.1.20:3005/org/repo") + with pytest.raises(RuntimeError, match="loopback or private-range"): + public_source_base() + + def test_mdns_name_is_rejected(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://pi5-16.local:3005/org/repo") + with pytest.raises(RuntimeError, match="mDNS"): + public_source_base() + + def test_rejection_names_the_variable_to_set(self, monkeypatch): + monkeypatch.setenv("GITEA_REPO_URL", "http://localhost:3005/org/repo") + with pytest.raises(RuntimeError, match="PUBLIC_URL_TO_GITEA"): + public_source_base() + + def test_reads_the_named_variable(self, monkeypatch): + monkeypatch.setenv("OTHER_BASE", "https://git.example.org/org/repo") + assert public_source_base("OTHER_BASE") == "https://git.example.org/org/repo"