"""Shared utilities for novoyuuparosk-auto-wiki pipelines.""" import html import os import re import subprocess import urllib.parse from pathlib import Path from urllib.parse import urlparse import mwclient AUTO_BANNER_PREFIX = "{{Auto-generated" # A ```columns fenced block is passed through Pandoc verbatim as #
…
; columns within it are separated by a line of ===. _COLUMNS_BLOCK_RE = re.compile(r'
(.*?)
', re.DOTALL) _COLUMN_SEP_RE = re.compile(r"^\s*===\s*$", re.MULTILINE) # Minimal Markdown emphasis → wikitext, for use inside a verbatim columns block # (Pandoc doesn't process Markdown there). Asterisk style only, single line. # Order matters: bold-italic (***) before bold (**) before italic (*). _BOLD_ITALIC_RE = re.compile(r"\*\*\*(.+?)\*\*\*") _BOLD_RE = re.compile(r"\*\*(.+?)\*\*") _ITALIC_RE = re.compile(r"\*(.+?)\*") # Pandoc's mediawiki writer renders a *relative* Markdown link target as a # wikilink carrying the raw path — [label](notes.md) becomes [[notes.md|label]] — # and an absolute URL as [url label]. Both shapes are matched here so that a # pandoc version which classifies relative targets the other way still resolves. # The alternation is scanned left to right, so a [[...]] is consumed whole and # the single-bracket branch never re-matches its interior. _MD_LINK_RE = re.compile( r"\[\[(?P[^\[\]|]+?)(?:\|(?P[^\[\]]*))?\]\]" r"|\[(?P[^\[\]\s|]+)(?:[ \t]+(?P[^\[\]]*))?\]" ) # Inline Markdown link target, read from the *source* body before Pandoc runs: # the "…](target)" tail of [label](target). Used for the link graph only. _MD_SOURCE_LINK_RE = re.compile(r"\]\(\s*\s]+)>?") # Anything still naming a .md file in a link position after resolution means the # renderer failed to understand it. That aborts the run; it never reaches the wiki. _UNRESOLVED_LINK_RES = ( ( re.compile(r"\[\[[^\[\]]*\.md(?:#[^\[\]]*)?(?:\|[^\[\]]*)?\]\]", re.IGNORECASE), "unresolved wikilink", ), ( re.compile(r"\[[^\[\]\s|]*\.md[^\[\]]*\]", re.IGNORECASE), "unresolved external-style link", ), ( re.compile(r"\]\([^()\s]*\.md[^()]*\)", re.IGNORECASE), "raw Markdown link (Markdown is not processed inside a ```columns fence)", ), ) def _md_emphasis_to_wikitext(text: str) -> str: text = _BOLD_ITALIC_RE.sub(r"'''''\1'''''", text) text = _BOLD_RE.sub(r"'''\1'''", text) text = _ITALIC_RE.sub(r"''\1''", text) return text def strip_first_h1(text: str) -> str: """Remove the first '# Heading' line and any immediately following blank line.""" lines = text.split("\n") for i, line in enumerate(lines): if line.strip().startswith("# "): del lines[i] if i < len(lines) and lines[i].strip() == "": del lines[i] break return "\n".join(lines) def expand_columns(wikitext: str) -> str: """Expand ```columns fenced blocks into a flex row of columns. Authors write a fenced code block tagged ``columns``; Pandoc passes its body through verbatim as ``
…
`` (line breaks and blank lines preserved, inline markup entity-escaped). Columns within the block are separated by a line containing only ``===``. Each column is wrapped in so its line breaks survive MediaWiki parsing. Content is HTML-unescaped (so inline markup such as … renders rather than appearing as literal text) and Markdown emphasis (``*italic*``, ``**bold**``, ``***bold-italic***``) is converted to wikitext, since Pandoc does not process Markdown inside the verbatim block. """ def render(match: re.Match) -> str: body = _md_emphasis_to_wikitext(html.unescape(match.group(1))) columns = _COLUMN_SEP_RE.split(body) poems = "".join("\n" + col.strip("\n") + "\n" for col in columns) return '
' + poems + "
" return _COLUMNS_BLOCK_RE.sub(render, wikitext) def extract_md_link_targets(markdown_body: str) -> list[str]: """Raw targets of inline Markdown links in an unrendered body. Used to build the link graph *before* Pandoc runs, so the publisher can work out which pages a changed page points at (and which point back) without rendering the whole repository. Targets are returned verbatim, including any ``#fragment`` and percent-encoding; the caller resolves them. """ return _MD_SOURCE_LINK_RE.findall(markdown_body) def assert_no_unresolved_md_links(wikitext: str, source_path: Path) -> None: """Abort if any link still names a .md file after resolution. Guards against a link shape the resolver does not recognise: a pandoc version that writes relative targets differently, or a Markdown link buried in a verbatim ```columns fence. Without this, such a link would reach the wiki as literal text, silently corrupting a published page. """ for pattern, kind in _UNRESOLVED_LINK_RES: found = pattern.search(wikitext) if found: raise ValueError( f"{source_path}: {kind} still references a .md file after " f"resolution: {found.group(0)!r}" ) def resolve_md_links(wikitext: str, source_path: Path, title_index: dict[str, str]) -> str: """Rewrite links to sibling ``.md`` files into wikilinks to their wiki pages. A related co-publication is nominated with an ordinary relative Markdown link — ``see [dev notes](pulse_dev.md)`` — which stays a working link in Gitea's blob view and in any Markdown preview. Here the ``.md`` target is resolved against ``title_index`` (absolute path string → wiki page title) and replaced with ``[[Wiki page title|label]]``. ``title_index`` must cover every publishable file in the repository, not only the batch being published, or the same link resolves differently depending on which files a push happened to touch. A ``.md`` target that cannot be resolved raises ``ValueError``: either the file does not exist, or it exists but is not published. Both are authoring mistakes, and both fail the run rather than silently degrade the page. Link targets that do not name a ``.md`` file are left exactly as Pandoc wrote them. """ def resolve(match: re.Match) -> str: groups = match.groupdict() if groups["wiki_target"] is not None: target, label = groups["wiki_target"], groups["wiki_label"] else: target, label = groups["ext_target"], groups["ext_label"] raw_path, _, fragment = target.partition("#") rel_path = urllib.parse.unquote(raw_path) if not rel_path.lower().endswith(".md"): return match.group(0) resolved = (source_path.parent / rel_path).resolve() title = title_index.get(str(resolved)) if title is None: reason = ( "file exists but is not published (no 'wiki.publish: true', " "or it lives under wip/)" if resolved.exists() else "file does not exist" ) raise ValueError( f"{source_path}: link to '{target}' cannot be resolved - {reason}" ) anchor = f"#{fragment}" if fragment else "" # A label that repeats the path (pandoc's form for an unlabelled link) or # repeats the page title needs no pipe: [[Title]] already displays it. if label is None or label.strip() in (target, title): return f"[[{title}{anchor}]]" return f"[[{title}{anchor}|{label}]]" resolved = _MD_LINK_RE.sub(resolve, wikitext) assert_no_unresolved_md_links(resolved, source_path) return resolved def markdown_to_wikitext(body: str) -> str: # encoding is pinned rather than left to text=True, which would decode # pandoc's output with the ambient locale encoding. The lyrics are Japanese # and Chinese; on any non-UTF-8 locale that decode dies inside subprocess's # reader thread, stdout comes back None, and the real error is lost. result = subprocess.run( ["pandoc", "-f", "markdown", "-t", "mediawiki"], input=body, capture_output=True, encoding="utf-8", ) if result.returncode != 0: raise RuntimeError(f"pandoc failed: {result.stderr.strip()}") return expand_columns(result.stdout) def connect_wiki() -> mwclient.Site: api_url = os.environ["WIKI_API_URL"] parsed = urlparse(api_url) host = parsed.netloc path = parsed.path[: parsed.path.rfind("/") + 1] or "/" site = mwclient.Site(host, path=path, scheme=parsed.scheme) site.login(os.environ["WIKI_BOT_USER"], os.environ["WIKI_BOT_PASSWORD"]) return site