songs: replace the wiki.siblings macro with plain Markdown links

The `wiki.siblings` bill of materials is gone. A related co-publication is
now nominated with an ordinary relative Markdown link — `[label](notes.md)` —
resolved at publish time against the target file's own `title` frontmatter.
No macro namespace, no per-file declaration, and the link still works in
Gitea's blob view and any editor preview.

The macro could not be made safe. Substitution was `str.replace(tag, …)` over
the whole rendered page: no word boundary, so a tag also occurring in lyrics or
a code block got rewritten; no collision check, so overlapping tags corrupted
each other in declaration order. SCHEMA.md documented the placeholder as
`[[TAG]]` while the code replaced the bare tag, so following the schema
produced `[[[[Title|label]]]]`.

Fixes a live corruption bug in the process. `sibling_map` was built only from
the files in the publish batch, and an unresolved sibling degraded to plain
text — so pushing `ses/pulse.md` alone republished it with its wikilink
silently stripped to the bare words "Dev gibberish". SCHEMA.md asserted the
workflow "always runs --all", which it never did on an ordinary push.

The repository is now always scanned in full, whatever a run publishes, so
link resolution and title-collision detection can never depend on which files
a push touched. Writes stay bounded to the push diff plus one hop in each link
direction: the page a changed page links to (so a link cannot point at an
unpublished page) and the pages linking to it (so an edited `title` does not
leave stale inbound links). Publishing is not instant; reading is cheap.

An unresolvable `.md` link is now a hard error, never graceful degradation, and
linking to a file still does not publish it — `wiki.publish: true` remains the
only gate. A leftover-link guard fails the run on any link shape the resolver
did not recognise, including a Markdown link written inside a verbatim
```columns fence, so a pandoc upgrade cannot silently publish literal text.

Also:

- render before write: every page in a run is rendered before the first is
  published, so a broken page leaves the wiki untouched instead of
  half-updated. `publish_one` is split into `render_page` (pure) and the write.
- pin pandoc's stdout to UTF-8. `text=True` decoded it with the ambient locale
  encoding; on any non-UTF-8 locale the Japanese and Chinese lyrics died inside
  subprocess's reader thread, `stdout` came back None, and the real error was
  lost behind a TypeError. Latent on the Debian runner via PEP 538 only.
- title collisions are detected repo-wide, not just within a push, and before
  any wiki contact.
- tests/, run by .gitea/workflows/test.yml in the publish runner image. The
  pandoc output contract the resolver depends on is now pinned by a test
  instead of by prose — which is how SCHEMA.md came to assert something false.
- drop an f-string with nothing to interpolate.
- quote the SOURCE_REF interpolation in the workflow instead of pasting an
  input straight into shell.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-09 16:16:33 +09:00
parent 4e7d84d665
commit f5625cde98
10 changed files with 838 additions and 183 deletions
+42
View File
@@ -0,0 +1,42 @@
import importlib.util
import sys
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))
def _load_pipeline(name: str):
"""Import a pipeline's publish.py under a unique module name.
The three pipelines each define a module called `publish`, so they cannot be
imported by plain name without colliding in sys.modules.
"""
spec = importlib.util.spec_from_file_location(
f"{name}_publish", REPO_ROOT / "pipelines" / name / "publish.py"
)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
@pytest.fixture(scope="session")
def songs():
return _load_pipeline("songs")
@pytest.fixture
def repo(tmp_path):
"""Build a source tree of .md files: write(relative_path, frontmatter, body)."""
def write(rel: str, front: str = "", body: str = "") -> Path:
path = tmp_path / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f"---\n{front}\n---\n\n{body}\n", encoding="utf-8")
return path
write.root = tmp_path
return write
+169
View File
@@ -0,0 +1,169 @@
"""Tests for the songs publisher: discovery, validation, and publish scoping.
None of these touch a wiki. `discover_publishable` and `select_publish_set` are
pure filesystem reads, which is the point — every failure they catch must fail
the run before `connect_wiki()` is ever called.
"""
import pytest
PUBLISHED = "title: {title}\nwiki:\n publish: true"
UNPUBLISHED = "title: {title}\nwiki:\n publish: false"
class TestDiscoverPublishable:
def test_collects_only_opted_in_files(self, songs, repo):
repo("a.md", PUBLISHED.format(title="A"))
repo("b.md", UNPUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
assert [p.metadata["title"] for p in posts.values()] == ["A"]
def test_skips_wip_directory_even_when_opted_in(self, songs, repo):
repo("a.md", PUBLISHED.format(title="A"))
repo("wip/draft.md", PUBLISHED.format(title="Draft"))
posts = songs.discover_publishable(repo.root)
assert [p.metadata["title"] for p in posts.values()] == ["A"]
def test_skips_readme(self, songs, repo):
repo("README.md", PUBLISHED.format(title="Readme"))
assert songs.discover_publishable(repo.root) == {}
def test_scans_nested_directories(self, songs, repo):
repo("ses/a.md", PUBLISHED.format(title="A"))
repo("kairo/deep/b.md", PUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
assert sorted(p.metadata["title"] for p in posts.values()) == ["A", "B"]
def test_duplicate_titles_in_different_directories_fail(self, songs, repo):
repo("ses/a.md", PUBLISHED.format(title="Same page"))
repo("kairo/b.md", PUBLISHED.format(title="Same page"))
with pytest.raises(ValueError, match="Duplicate wiki title 'Same page'"):
songs.discover_publishable(repo.root)
def test_duplicate_title_error_names_both_files(self, songs, repo):
repo("ses/a.md", PUBLISHED.format(title="Same page"))
repo("kairo/b.md", PUBLISHED.format(title="Same page"))
with pytest.raises(ValueError) as err:
songs.discover_publishable(repo.root)
assert "a.md" in str(err.value) and "b.md" in str(err.value)
def test_a_title_colliding_with_an_unpublished_file_is_allowed(self, songs, repo):
repo("a.md", PUBLISHED.format(title="Same page"))
repo("b.md", UNPUBLISHED.format(title="Same page"))
assert len(songs.discover_publishable(repo.root)) == 1
def test_publish_as_a_string_is_rejected(self, songs, repo):
repo("a.md", 'title: A\nwiki:\n publish: "true"')
with pytest.raises(ValueError, match="must be a boolean"):
songs.discover_publishable(repo.root)
def test_missing_title_is_rejected(self, songs, repo):
repo("a.md", "wiki:\n publish: true")
with pytest.raises(ValueError, match="missing or empty 'title'"):
songs.discover_publishable(repo.root)
def test_bad_release_date_is_rejected(self, songs, repo):
repo("a.md", PUBLISHED.format(title="A") + "\nrelease_date: 14-06-2026")
with pytest.raises(ValueError, match="YYYY-MM-DD"):
songs.discover_publishable(repo.root)
def test_categories_must_be_a_list(self, songs, repo):
repo("a.md", PUBLISHED.format(title="A") + "\ncategories: kairo")
with pytest.raises(ValueError, match="'categories' must be a list"):
songs.discover_publishable(repo.root)
def test_a_leftover_siblings_key_is_ignored_not_an_error(self, songs, repo):
# The old macro declaration is inert; a stale one must not fail a run.
repo("a.md", PUBLISHED.format(title="A") + "\n siblings:\n - tag: X")
assert len(songs.discover_publishable(repo.root)) == 1
class TestSelectPublishSet:
def test_publishes_only_the_changed_file(self, songs, repo):
a = repo("a.md", PUBLISHED.format(title="A"))
repo("b.md", PUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
selected = songs.select_publish_set(posts, {a.resolve()})
assert [p.metadata["title"] for p in selected.values()] == ["A"]
def test_arms_the_page_a_changed_page_links_to(self, songs, repo):
a = repo("a.md", PUBLISHED.format(title="A"), "see [notes](b.md)")
repo("b.md", PUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
selected = songs.select_publish_set(posts, {a.resolve()})
assert sorted(p.metadata["title"] for p in selected.values()) == ["A", "B"]
def test_arms_the_page_that_links_to_a_changed_page(self, songs, repo):
repo("a.md", PUBLISHED.format(title="A"), "see [notes](b.md)")
b = repo("b.md", PUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
selected = songs.select_publish_set(posts, {b.resolve()})
assert sorted(p.metadata["title"] for p in selected.values()) == ["A", "B"]
def test_unrelated_pages_stay_out(self, songs, repo):
a = repo("a.md", PUBLISHED.format(title="A"), "see [notes](b.md)")
repo("b.md", PUBLISHED.format(title="B"))
repo("c.md", PUBLISHED.format(title="C"))
posts = songs.discover_publishable(repo.root)
selected = songs.select_publish_set(posts, {a.resolve()})
assert "C" not in [p.metadata["title"] for p in selected.values()]
def test_links_across_directories_are_followed(self, songs, repo):
a = repo("ses/a.md", PUBLISHED.format(title="A"), "see [notes](../kairo/b.md)")
repo("kairo/b.md", PUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
selected = songs.select_publish_set(posts, {a.resolve()})
assert sorted(p.metadata["title"] for p in selected.values()) == ["A", "B"]
def test_a_link_to_an_unpublished_file_adds_nothing(self, songs, repo):
# resolve_md_links is what rejects this loudly; the graph just drops it.
a = repo("a.md", PUBLISHED.format(title="A"), "see [draft](b.md)")
repo("b.md", UNPUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
selected = songs.select_publish_set(posts, {a.resolve()})
assert [p.metadata["title"] for p in selected.values()] == ["A"]
def test_a_self_link_does_not_duplicate(self, songs, repo):
a = repo("a.md", PUBLISHED.format(title="A"), "see [self](a.md)")
posts = songs.discover_publishable(repo.root)
selected = songs.select_publish_set(posts, {a.resolve()})
assert len(selected) == 1
class TestBuildTitleIndex:
def test_maps_every_publishable_path_to_its_title(self, songs, repo):
a = repo("a.md", PUBLISHED.format(title="A"))
posts = songs.discover_publishable(repo.root)
assert songs.build_title_index(posts)[str(a.resolve())] == "A"
def test_covers_the_whole_repo_not_just_a_changed_file(self, songs, repo):
repo("a.md", PUBLISHED.format(title="A"))
b = repo("nested/b.md", PUBLISHED.format(title="B"))
posts = songs.discover_publishable(repo.root)
assert str(b.resolve()) in songs.build_title_index(posts)
class TestBuildWikitext:
def test_album_injects_both_song_and_album_categories(self, songs):
out = songs.build_wikitext({"album": "kairo II"}, "body\n", "url", "sha")
assert "[[Category:Songs]]" in out
assert "[[Category:Album:kairo II]]" in out
def test_no_album_injects_no_category(self, songs):
out = songs.build_wikitext({}, "body\n", "url", "sha")
assert "[[Category:" not in out
def test_extra_categories_are_additive(self, songs):
out = songs.build_wikitext(
{"album": "kairo II", "categories": ["kairo"]}, "body\n", "url", "sha"
)
assert "[[Category:kairo]]" in out
assert "[[Category:Album:kairo II]]" in out
def test_banner_carries_source_and_commit(self, songs):
out = songs.build_wikitext({}, "body\n", "http://g/x", "abc123")
assert out.startswith("{{Auto-generated|source=http://g/x|commit=abc123}}")
def test_lrc_footer_link_is_appended_when_named(self, songs):
out = songs.build_wikitext({}, "body\n", "url", "sha", "A.lrc")
assert "[[Media:A.lrc|Synced lyrics (.lrc)]]" in out
+216
View File
@@ -0,0 +1,216 @@
"""Tests for the shared render helpers in lib/wiki.py.
All pure string in, string out, except the pandoc contract test at the bottom,
which is the canary for a pandoc upgrade changing how relative links are written.
"""
import shutil
import subprocess
from pathlib import Path
import pytest
from lib.wiki import (
assert_no_unresolved_md_links,
expand_columns,
extract_md_link_targets,
markdown_to_wikitext,
resolve_md_links,
strip_first_h1,
)
# Paths need not exist on disk: resolution is non-strict, and only the missing
# vs. unpublished distinction consults the filesystem (covered with tmp_path).
SONGS = Path(__file__).resolve().parent / "_paths" / "songs"
HERE = SONGS / "pulse.md"
INDEX = {
str((SONGS / "pulse.md").resolve()): "pulse under latent semantic envelopes",
str((SONGS / "pulse_dev.md").resolve()): (
"pulse under latent semantic envelopes - development notes"
),
str((SONGS / "two words.md").resolve()): "Two words",
}
def resolve(wikitext, source=HERE):
return resolve_md_links(wikitext, source, INDEX)
class TestStripFirstH1:
def test_removes_heading_and_following_blank_line(self):
assert strip_first_h1("# Title\n\nbody") == "body"
def test_removes_only_the_first(self):
assert strip_first_h1("# One\n\n# Two\n") == "# Two\n"
def test_leaves_body_without_h1_alone(self):
assert strip_first_h1("## Sub\n\nbody") == "## Sub\n\nbody"
def test_finds_the_h1_even_when_a_deeper_heading_precedes_it(self):
assert strip_first_h1("### Deep\n\n# Title\n") == "### Deep\n"
class TestExpandColumns:
def test_splits_on_separator_and_wraps_each_in_poem(self):
out = expand_columns('<pre class="columns">a\nb\n===\nc</pre>')
assert out.count("<poem>") == 2
assert "<poem>\na\nb\n</poem>" in out
assert "<poem>\nc\n</poem>" in out
assert out.startswith("<div style=")
def test_three_columns_need_two_separators(self):
out = expand_columns('<pre class="columns">a\n===\nb\n===\nc</pre>')
assert out.count("<poem>") == 3
def test_converts_markdown_emphasis(self):
out = expand_columns('<pre class="columns">***a*** **b** *c*</pre>')
assert "'''''a'''''" in out
assert "'''b'''" in out
assert "''c''" in out
def test_unescapes_inline_html(self):
out = expand_columns('<pre class="columns">&lt;b&gt;x&lt;/b&gt;</pre>')
assert "<b>x</b>" in out
def test_leaves_wikitext_without_a_columns_block_untouched(self):
assert expand_columns("plain '''wikitext'''") == "plain '''wikitext'''"
class TestResolveMdLinks:
def test_labelled_link_becomes_piped_wikilink(self):
assert resolve("see [[pulse_dev.md|dev notes]] ok") == (
"see [[pulse under latent semantic envelopes - development notes|dev notes]] ok"
)
def test_unlabelled_link_uses_the_page_title(self):
assert resolve("[[pulse_dev.md]]") == (
"[[pulse under latent semantic envelopes - development notes]]"
)
def test_label_identical_to_target_drops_the_pipe(self):
assert resolve("[[pulse_dev.md|pulse_dev.md]]") == (
"[[pulse under latent semantic envelopes - development notes]]"
)
def test_label_identical_to_the_page_title_drops_the_pipe(self):
assert resolve("[[pulse_dev.md|pulse under latent semantic envelopes - development notes]]") == (
"[[pulse under latent semantic envelopes - development notes]]"
)
def test_fragment_is_carried_onto_the_title(self):
assert resolve("[[pulse_dev.md#Notes|x]]") == (
"[[pulse under latent semantic envelopes - development notes#Notes|x]]"
)
def test_percent_encoding_in_target_is_decoded(self):
assert resolve("[[two%20words.md|x]]") == "[[Two words|x]]"
def test_parent_directory_target_resolves(self):
assert resolve_md_links(
"[[../pulse_dev.md|x]]", SONGS / "sub" / "a.md", INDEX
) == "[[pulse under latent semantic envelopes - development notes|x]]"
def test_external_style_link_shape_also_resolves(self):
# Insurance against a pandoc version writing relative targets this way.
assert resolve("[pulse_dev.md dev notes]") == (
"[[pulse under latent semantic envelopes - development notes|dev notes]]"
)
def test_absolute_url_is_left_alone(self):
assert resolve("[https://example.com/x label]") == "[https://example.com/x label]"
def test_non_md_wikilink_is_left_alone(self):
assert resolve("[[Category:Songs]] [[Media:x.lrc|y]]") == (
"[[Category:Songs]] [[Media:x.lrc|y]]"
)
def test_label_containing_a_pipe_is_preserved(self):
assert resolve("[[pulse_dev.md|a|b]]").endswith("|a|b]]")
def test_several_links_in_one_body(self):
out = resolve("[[pulse_dev.md|a]] then [[two%20words.md|b]]")
assert "|a]]" in out and "[[Two words|b]]" in out
def test_missing_file_is_an_error(self):
with pytest.raises(ValueError, match="does not exist"):
resolve("[[nope.md|x]]")
def test_existing_but_unpublished_file_is_an_error(self, tmp_path):
(tmp_path / "draft.md").write_text("x", encoding="utf-8")
with pytest.raises(ValueError, match="not published"):
resolve_md_links("[[draft.md|x]]", tmp_path / "song.md", {})
def test_error_names_the_source_file(self):
with pytest.raises(ValueError, match=r"pulse\.md"):
resolve("[[nope.md|x]]")
class TestUnresolvedLinkGuard:
def test_accepts_output_with_no_md_reference(self):
assert_no_unresolved_md_links("[[A page|x]] and [http://e.com y]", HERE)
def test_rejects_leftover_wikilink(self):
with pytest.raises(ValueError, match="unresolved wikilink"):
assert_no_unresolved_md_links("[[notes.md|x]]", HERE)
def test_rejects_leftover_external_style_link(self):
with pytest.raises(ValueError, match="external-style"):
assert_no_unresolved_md_links("[notes.md x]", HERE)
def test_rejects_raw_markdown_link_left_in_a_columns_fence(self):
with pytest.raises(ValueError, match="columns"):
assert_no_unresolved_md_links("<poem>\nsee [x](notes.md)\n</poem>", HERE)
def test_prose_mentioning_a_filename_is_not_a_link(self):
assert_no_unresolved_md_links("edit <code>pulse.md</code> by hand", HERE)
class TestExtractMdLinkTargets:
def test_finds_inline_targets(self):
assert extract_md_link_targets("a [x](one.md) b [y](../two.md#f)") == [
"one.md",
"../two.md#f",
]
def test_ignores_bodies_without_links(self):
assert extract_md_link_targets("no links here") == []
def test_finds_targets_inside_a_columns_fence(self):
# The fence cannot render them, but the graph must still see the edge.
assert extract_md_link_targets("```columns\n[x](one.md)\n```") == ["one.md"]
@pytest.mark.skipif(shutil.which("pandoc") is None, reason="pandoc not installed")
class TestPandocContract:
"""Pins the pandoc output shape that resolve_md_links depends on.
If a pandoc upgrade starts writing relative link targets differently, this
fails here instead of silently publishing a broken link.
"""
def test_version(self):
out = subprocess.run(
["pandoc", "--version"], capture_output=True, text=True, check=True
).stdout
print(out.splitlines()[0])
def test_relative_link_survives_the_full_render(self):
wikitext = markdown_to_wikitext("see [dev notes](pulse_dev.md) here")
assert resolve(wikitext).strip() == (
"see [[pulse under latent semantic envelopes - development notes|dev notes]] here"
)
def test_unlabelled_relative_link_survives_the_full_render(self):
wikitext = markdown_to_wikitext("[pulse_dev.md](pulse_dev.md)")
assert resolve(wikitext).strip() == (
"[[pulse under latent semantic envelopes - development notes]]"
)
def test_absolute_url_survives_untouched(self):
wikitext = markdown_to_wikitext("[site](https://example.com/x)")
assert resolve(wikitext).strip() == "[https://example.com/x site]"
def test_columns_fence_renders_to_side_by_side_poems(self):
wikitext = markdown_to_wikitext("```columns\n**A**\nl1\n===\n**B**\nl2\n```")
assert wikitext.count("<poem>") == 2
assert "'''A'''" in wikitext