"""Shared utilities for novoyuuparosk-auto-wiki pipelines."""
import html
import os
import re
import subprocess
import urllib.parse
from pathlib import Path
from urllib.parse import urlparse
import mwclient
AUTO_BANNER_PREFIX = "{{Auto-generated"
# A ```columns fenced block is passed through Pandoc verbatim as
#
…
; columns within it are separated by a line of ===.
_COLUMNS_BLOCK_RE = re.compile(r'(.*?)
', re.DOTALL)
_COLUMN_SEP_RE = re.compile(r"^\s*===\s*$", re.MULTILINE)
# Minimal Markdown emphasis → wikitext, for use inside a verbatim columns block
# (Pandoc doesn't process Markdown there). Asterisk style only, single line.
# Order matters: bold-italic (***) before bold (**) before italic (*).
_BOLD_ITALIC_RE = re.compile(r"\*\*\*(.+?)\*\*\*")
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
_ITALIC_RE = re.compile(r"\*(.+?)\*")
# Pandoc's mediawiki writer renders a *relative* Markdown link target as a
# wikilink carrying the raw path — [label](notes.md) becomes [[notes.md|label]] —
# and an absolute URL as [url label]. Both shapes are matched here so that a
# pandoc version which classifies relative targets the other way still resolves.
# The alternation is scanned left to right, so a [[...]] is consumed whole and
# the single-bracket branch never re-matches its interior.
_MD_LINK_RE = re.compile(
r"\[\[(?P[^\[\]|]+?)(?:\|(?P[^\[\]]*))?\]\]"
r"|\[(?P[^\[\]\s|]+)(?:[ \t]+(?P[^\[\]]*))?\]"
)
# Inline Markdown link target, read from the *source* body before Pandoc runs:
# the "…](target)" tail of [label](target). Used for the link graph only.
_MD_SOURCE_LINK_RE = re.compile(r"\]\(\s*([^()<>\s]+)>?")
# Anything still naming a .md file in a link position after resolution means the
# renderer failed to understand it. That aborts the run; it never reaches the wiki.
_UNRESOLVED_LINK_RES = (
(
re.compile(r"\[\[[^\[\]]*\.md(?:#[^\[\]]*)?(?:\|[^\[\]]*)?\]\]", re.IGNORECASE),
"unresolved wikilink",
),
(
re.compile(r"\[[^\[\]\s|]*\.md[^\[\]]*\]", re.IGNORECASE),
"unresolved external-style link",
),
(
re.compile(r"\]\([^()\s]*\.md[^()]*\)", re.IGNORECASE),
"raw Markdown link (Markdown is not processed inside a ```columns fence)",
),
)
def _md_emphasis_to_wikitext(text: str) -> str:
text = _BOLD_ITALIC_RE.sub(r"'''''\1'''''", text)
text = _BOLD_RE.sub(r"'''\1'''", text)
text = _ITALIC_RE.sub(r"''\1''", text)
return text
def strip_first_h1(text: str) -> str:
"""Remove the first '# Heading' line and any immediately following blank line."""
lines = text.split("\n")
for i, line in enumerate(lines):
if line.strip().startswith("# "):
del lines[i]
if i < len(lines) and lines[i].strip() == "":
del lines[i]
break
return "\n".join(lines)
def expand_columns(wikitext: str) -> str:
"""Expand ```columns fenced blocks into a flex row of columns.
Authors write a fenced code block tagged ``columns``; Pandoc passes its body
through verbatim as ``…
`` (line breaks and blank
lines preserved, inline markup entity-escaped). Columns within the block are
separated by a line containing only ``===``. Each column is wrapped in
so its line breaks survive MediaWiki parsing. Content is
HTML-unescaped (so inline markup such as … renders rather than
appearing as literal text) and Markdown emphasis (``*italic*``, ``**bold**``,
``***bold-italic***``) is converted to wikitext, since Pandoc does not
process Markdown inside the verbatim block.
"""
def render(match: re.Match) -> str:
body = _md_emphasis_to_wikitext(html.unescape(match.group(1)))
columns = _COLUMN_SEP_RE.split(body)
poems = "".join("\n" + col.strip("\n") + "\n" for col in columns)
return '' + poems + "
"
return _COLUMNS_BLOCK_RE.sub(render, wikitext)
def extract_md_link_targets(markdown_body: str) -> list[str]:
"""Raw targets of inline Markdown links in an unrendered body.
Used to build the link graph *before* Pandoc runs, so the publisher can work
out which pages a changed page points at (and which point back) without
rendering the whole repository. Targets are returned verbatim, including any
``#fragment`` and percent-encoding; the caller resolves them.
"""
return _MD_SOURCE_LINK_RE.findall(markdown_body)
def assert_no_unresolved_md_links(wikitext: str, source_path: Path) -> None:
"""Abort if any link still names a .md file after resolution.
Guards against a link shape the resolver does not recognise: a pandoc version
that writes relative targets differently, or a Markdown link buried in a
verbatim ```columns fence. Without this, such a link would reach the wiki as
literal text, silently corrupting a published page.
"""
for pattern, kind in _UNRESOLVED_LINK_RES:
found = pattern.search(wikitext)
if found:
raise ValueError(
f"{source_path}: {kind} still references a .md file after "
f"resolution: {found.group(0)!r}"
)
def resolve_md_links(wikitext: str, source_path: Path, title_index: dict[str, str]) -> str:
"""Rewrite links to sibling ``.md`` files into wikilinks to their wiki pages.
A related co-publication is nominated with an ordinary relative Markdown
link — ``see [dev notes](pulse_dev.md)`` — which stays a working link in
Gitea's blob view and in any Markdown preview. Here the ``.md`` target is
resolved against ``title_index`` (absolute path string → wiki page title) and
replaced with ``[[Wiki page title|label]]``.
``title_index`` must cover every publishable file in the repository, not only
the batch being published, or the same link resolves differently depending on
which files a push happened to touch.
A ``.md`` target that cannot be resolved raises ``ValueError``: either the
file does not exist, or it exists but is not published. Both are authoring
mistakes, and both fail the run rather than silently degrade the page. Link
targets that do not name a ``.md`` file are left exactly as Pandoc wrote them.
"""
def resolve(match: re.Match) -> str:
groups = match.groupdict()
if groups["wiki_target"] is not None:
target, label = groups["wiki_target"], groups["wiki_label"]
else:
target, label = groups["ext_target"], groups["ext_label"]
raw_path, _, fragment = target.partition("#")
rel_path = urllib.parse.unquote(raw_path)
if not rel_path.lower().endswith(".md"):
return match.group(0)
resolved = (source_path.parent / rel_path).resolve()
title = title_index.get(str(resolved))
if title is None:
reason = (
"file exists but is not published (no 'wiki.publish: true', "
"or it lives under wip/)"
if resolved.exists()
else "file does not exist"
)
raise ValueError(
f"{source_path}: link to '{target}' cannot be resolved - {reason}"
)
anchor = f"#{fragment}" if fragment else ""
# A label that repeats the path (pandoc's form for an unlabelled link) or
# repeats the page title needs no pipe: [[Title]] already displays it.
if label is None or label.strip() in (target, title):
return f"[[{title}{anchor}]]"
return f"[[{title}{anchor}|{label}]]"
resolved = _MD_LINK_RE.sub(resolve, wikitext)
assert_no_unresolved_md_links(resolved, source_path)
return resolved
def markdown_to_wikitext(body: str) -> str:
# encoding is pinned rather than left to text=True, which would decode
# pandoc's output with the ambient locale encoding. The lyrics are Japanese
# and Chinese; on any non-UTF-8 locale that decode dies inside subprocess's
# reader thread, stdout comes back None, and the real error is lost.
result = subprocess.run(
["pandoc", "-f", "markdown", "-t", "mediawiki"],
input=body,
capture_output=True,
encoding="utf-8",
)
if result.returncode != 0:
raise RuntimeError(f"pandoc failed: {result.stderr.strip()}")
return expand_columns(result.stdout)
def connect_wiki() -> mwclient.Site:
api_url = os.environ["WIKI_API_URL"]
parsed = urlparse(api_url)
host = parsed.netloc
path = parsed.path[: parsed.path.rfind("/") + 1] or "/"
site = mwclient.Site(host, path=path, scheme=parsed.scheme)
site.login(os.environ["WIKI_BOT_USER"], os.environ["WIKI_BOT_PASSWORD"])
return site