#!/usr/bin/env python3 """Songs pipeline — validates, renders, and publishes song .md files to MediaWiki.""" import argparse import hashlib import io import os import sys from datetime import datetime from pathlib import Path import frontmatter import mwclient sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) from lib.wiki import AUTO_BANNER_PREFIX, connect_wiki, markdown_to_wikitext, strip_first_h1 def validate(fm: dict, path: Path) -> None: errors = [] val = fm.get("title") if not val or not str(val).strip(): errors.append("missing or empty 'title'") if not isinstance(fm.get("wiki", {}).get("publish"), bool): errors.append("'wiki.publish' must be a boolean (true/false), not a string") if "release_date" in fm and fm["release_date"]: try: datetime.strptime(str(fm["release_date"]), "%Y-%m-%d") except ValueError: errors.append("'release_date' must be YYYY-MM-DD") if "categories" in fm and not isinstance(fm["categories"], list): errors.append("'categories' must be a list") for i, sib in enumerate(fm.get("wiki", {}).get("siblings") or []): if not isinstance(sib, dict): errors.append(f"wiki.siblings[{i}] must be a mapping") continue sib_path = sib.get("path") sib_tag = sib.get("tag") if sib_path and sib_tag: if not (path.parent / str(sib_path)).resolve().exists(): errors.append(f"wiki.siblings[{i}]: path '{sib_path}' not found") if errors: raise ValueError(f"{path}: " + "; ".join(errors)) def build_sibling_map(posts: dict) -> dict[str, str]: """Map abs_path_str → wiki_title for all published posts.""" return {str(path): str(post.metadata["title"]) for path, post in posts.items()} def apply_sibling_substitutions(wikitext: str, this_path: Path, fm_wiki: dict, sibling_map: dict[str, str]) -> str: """Replace [[TAG]] placeholders using the declared sibling bill of materials.""" for sib in (fm_wiki.get("siblings") or []): if not isinstance(sib, dict): continue tag = sib.get("tag") path = sib.get("path") if not tag or not path: continue label = (sib.get("label") or "").strip() abs_sib = str((this_path.parent / str(path)).resolve()) title = sibling_map.get(abs_sib) if title and label: replacement = f"[[{title}|{label}]]" elif title: replacement = f"[[{title}]]" elif label: replacement = label else: replacement = tag wikitext = wikitext.replace(tag, replacement) return wikitext def build_wikitext(fm: dict, body_wikitext: str, source_url: str, source_ref: str, lrc_filename: str | None = None) -> str: banner = f"{{{{Auto-generated|source={source_url}|commit={source_ref}}}}}" lrc_line = f"[[Media:{lrc_filename}|Synced lyrics (.lrc)]]\n" if lrc_filename else "" cat_parts = [] if fm.get("album"): cat_parts.append(f"[[Category:Songs]]") cat_parts.append(f"[[Category:Album:{fm['album']}]]") for cat in (fm.get("categories") or []): cat_parts.append(f"[[Category:{cat}]]") categories = "\n".join(cat_parts) + "\n" if cat_parts else "" return f"{banner}\n\n{body_wikitext}{lrc_line}{categories}" def upload_lrc(lrc_path: Path, title: str, site: mwclient.Site) -> bool: """Upload LRC file to wiki if content has changed. Returns True if uploaded.""" if not lrc_path.exists(): print(f" warn: LRC file not found: {lrc_path}", file=sys.stderr) return False content = lrc_path.read_bytes() try: content.decode("utf-8") except UnicodeDecodeError: print(f" warn: {lrc_path} is not valid UTF-8 text, skipping LRC upload", file=sys.stderr) return False wiki_filename = f"{title}.lrc" local_sha1 = hashlib.sha1(content).hexdigest() if site.images[wiki_filename].imageinfo.get("sha1") == local_sha1: return False site.upload( file=io.BytesIO(content), filename=wiki_filename, description=f"Synced lyrics for [[{title}]] (auto-published)", ignore=True, ) return True def publish_one(abs_path: Path, source_dir: Path, post, site: mwclient.Site, source_ref: str, gitea_repo_url: str, sibling_map: dict[str, str]) -> str: """Returns 'published', 'noop', or 'skipped-manual'.""" rel = abs_path.relative_to(source_dir) fm = post.metadata title = str(fm["title"]) body_wikitext = markdown_to_wikitext(strip_first_h1(post.content)) body_wikitext = apply_sibling_substitutions(body_wikitext, abs_path, fm.get("wiki", {}), sibling_map) source_url = f"{gitea_repo_url}/src/commit/{source_ref}/{rel}" lrc_filename = None lrc_rel = fm.get("lrc") if lrc_rel: lrc_uploaded = upload_lrc(abs_path.parent / str(lrc_rel), title, site) lrc_filename = f"{title}.lrc" if lrc_uploaded: print(f" [lrc-uploaded] {lrc_filename}") page_content = build_wikitext(fm, body_wikitext, source_url, source_ref, lrc_filename) page = site.pages[title] existing = page.text() if existing and not existing.startswith(AUTO_BANNER_PREFIX): return "skipped-manual" if existing == page_content: return "noop" ref_short = source_ref[:8] if source_ref else "unknown" page.save(page_content, summary=f"Auto-published from {ref_short} (songs pipeline)") return "published" def load_and_validate(files: list[Path], source_dir: Path) -> dict[Path, object]: posts = {} titles: dict[str, Path] = {} for path in files: if not path.exists(): print(f" warn: {path} not found, skipping", file=sys.stderr) continue post = frontmatter.load(str(path)) wiki = post.metadata.get("wiki") if not isinstance(wiki, dict) or not wiki.get("publish"): continue validate(post.metadata, path) title = str(post.metadata["title"]) if title in titles: raise ValueError( f"Duplicate wiki title '{title}': {path} and {titles[title]}" ) titles[title] = path posts[path] = post return posts def main() -> None: parser = argparse.ArgumentParser(description="Publish song .md files to MediaWiki.") parser.add_argument("--source-dir", required=True, type=Path) parser.add_argument("--files", nargs="*", default=[], help="Relative paths within source-dir") parser.add_argument("--all", action="store_true", help="Publish all publishable files") args = parser.parse_args() source_dir = args.source_dir.resolve() source_ref = os.environ.get("SOURCE_REF", "") gitea_repo_url = os.environ.get("GITEA_REPO_URL", "").rstrip("/") if args.all: candidates = list(source_dir.rglob("*.md")) elif args.files: candidates = [source_dir / f for f in args.files] else: print("Nothing to do: pass --files or --all.") return files = [ p for p in candidates if p.exists() and p.suffix == ".md" and "wip" not in p.relative_to(source_dir).parts and p.name != "README.md" ] if not files: print("No publishable candidates after filtering.") return print(f"Validating {len(files)} file(s)...") posts = load_and_validate(files, source_dir) sibling_map = build_sibling_map(posts) print("Validation passed.") site = connect_wiki() counts = {"published": 0, "noop": 0, "skipped-manual": 0} for abs_path, post in posts.items(): rel = abs_path.relative_to(source_dir) outcome = publish_one(abs_path, source_dir, post, site, source_ref, gitea_repo_url, sibling_map) counts[outcome] += 1 if outcome == "skipped-manual": print(f" [skipped-manual] {rel} ← page exists without auto-gen banner; delete or add banner to hand over to bot", file=sys.stderr) else: print(f" [{outcome}] {rel}") print( f"\nDone: {counts['published']} published, {counts['noop']} unchanged, " f"{counts['skipped-manual']} skipped (existing manual pages)." ) if __name__ == "__main__": try: main() except Exception as e: print(f"ERROR: {e}", file=sys.stderr) sys.exit(1)