"""Generate a bill of materials for a song's untracked binary assets. The repo's guideline says every song keeps a ``BOM.md`` (or ``BOM.yaml``) listing the untracked input assets and explaining what each file is. This walks the conventional asset directories, prompts for a description per file, and writes the result. Stdlib only: the YAML is emitted by hand. If PyYAML happens to be installed it is used to *validate* the output, but it is never required. Usage:: python3 scripts/bom_gen.py # walk ./assets, write BOM.yaml python3 scripts/bom_gen.py path/to/song # walk a song directory python3 scripts/bom_gen.py --format md # write BOM.md instead python3 scripts/bom_gen.py --descriptions d.toml # non-interactive python3 scripts/bom_gen.py --describe-one audio/song.flac """ from __future__ import annotations import argparse import hashlib import os import re import sys from dataclasses import dataclass, field if __package__ in (None, ""): sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _common import ( # type: ignore[import-not-found] Die, Report, add_common_arguments, human_size, main_wrapper, text_table, ) else: # pragma: no cover from ._common import ( Die, Report, add_common_arguments, human_size, main_wrapper, text_table, ) #: Directories that hold untracked input assets, with the role we assume. ASSET_DIRS: tuple[tuple[str, str], ...] = ( ("audio", "Song audio: final mix, instrumental, stems"), ("images", "Cover art, stills, background plates"), ("fonts", "Typefaces used by the subtitles or overlays"), ("videos", "Material video clips"), ) #: File extensions that are worth flagging as (probably) not-yet-described. _DESCRIPTION_PLACEHOLDER = "" _BOM_MARKER = "# generated by scripts/bom_gen.py" # --------------------------------------------------------------------------- # # YAML emission (hand-rolled, stdlib only) # --------------------------------------------------------------------------- # #: Characters that force a value to be quoted in YAML. _NEEDS_QUOTES = re.compile(r"""[:\{\}\[\],&*#?|<>=!%@`"']|^\s|\s$|^$|^[-?]""") def yaml_scalar(value: str) -> str: """Quote *value* for YAML only when it needs it.""" if value.isdigit(): return f'"{value}"' # keep hashes/filenames that look numeric as strings if _NEEDS_QUOTES.search(value): escaped = value.replace("\\", "\\\\").replace('"', '\\"') return f'"{escaped}"' return value def yaml_key(value: str) -> str: """Quote a key that would otherwise be ambiguous (spaces, colons, leading -).""" if _NEEDS_QUOTES.search(value) or value.startswith("-") or value != value.strip(): escaped = value.replace("\\", "\\\\").replace('"', '\\"') return f'"{escaped}"' return value # --------------------------------------------------------------------------- # # Asset discovery # --------------------------------------------------------------------------- # @dataclass class Asset: path: str # relative to the BOM root, POSIX separators size: int description: str = "" digest: str = "" # short content hash, empty when --hash was not requested extra: dict[str, str] = field(default_factory=dict) @property def directory(self) -> str: return os.path.dirname(self.path).split("/")[0] or "." def file_digest(full: str, *, length: int = 12, chunk: int = 1 << 20) -> str: """Short BLAKE2b content hash, streamed so large assets do not land in RAM. Assets here are audio and video files that can run to hundreds of megabytes, so this reads in chunks rather than slurping the whole file. BLAKE2b is in the standard library and is faster than md5 with no collision worries. ``length`` bytes of digest = 3 * length hex characters. """ h = hashlib.blake2b(digest_size=length) with open(full, "rb") as fh: while bytes_read := fh.read(chunk): h.update(bytes_read) return h.hexdigest() def walk_assets( root: str, dirs: tuple[tuple[str, str], ...] = ASSET_DIRS, *, hashes: bool = False, ) -> list[Asset]: """Collect files under each asset directory, sorted by path. Hashes are opt-in: they cost a full read of every asset, which is wasted work when you only want the inventory. """ assets: list[Asset] = [] for name, _role in dirs: base = os.path.join(root, name) if not os.path.isdir(base): continue for dirpath, dirnames, filenames in os.walk(base): dirnames.sort() for filename in sorted(filenames): if filename.startswith("."): continue full = os.path.join(dirpath, filename) if not os.path.isfile(full): continue rel = os.path.relpath(full, root).replace(os.sep, "/") assets.append( Asset( path=rel, size=os.path.getsize(full), digest=file_digest(full) if hashes else "", ) ) return assets def load_descriptions(path: str) -> dict[str, str]: """Read a descriptions file: simple ``path = description`` or ``pathdesc``. A deliberately forgiving format so descriptions can also be pasted in from a spreadsheet. ``#`` starts a comment. """ try: with open(path, "r", encoding="utf-8") as handle: raw = handle.read() except OSError as exc: raise Die(f"cannot read --descriptions {path}: {exc.strerror}") from exc mapping: dict[str, str] = {} for line in raw.splitlines(): line = line.strip() if not line or line.startswith("#"): continue if "=" in line: key, _, value = line.partition("=") elif "\t" in line: key, _, value = line.partition("\t") else: continue mapping[key.strip().lstrip("./")] = value.strip() return mapping # --------------------------------------------------------------------------- # # Rendering # --------------------------------------------------------------------------- # def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bool) -> str: total = sum(a.size for a in assets) lines = [ "---", _BOM_MARKER, f"root: {yaml_scalar(os.path.basename(os.path.abspath(root)) or root)}", ] if title: lines.append(f"title: {yaml_scalar(title)}") lines.append(f"total_files: {len(assets)}") lines.append(f"total_size: {total}") lines.append("") if by_dir: for dirname, role in ASSET_DIRS: grouping = [a for a in assets if a.directory == dirname] if not grouping: continue subtotal = sum(a.size for a in grouping) lines.append(f"{yaml_key(dirname)}:") lines.append(f" role: {yaml_scalar(role)}") lines.append(f" count: {len(grouping)}") lines.append(f" size: {subtotal}") lines.append(" files:") for asset in grouping: lines.append(f" {yaml_key(asset.path)}:") lines.append(f" size: {asset.size}") if asset.digest: lines.append(f" hash: {asset.digest}") lines.append(f" description: {yaml_scalar(asset.description)}") lines.append("") else: lines.append("files:") for asset in assets: lines.append(f" {yaml_key(asset.path)}:") lines.append(f" size: {asset.size}") if asset.digest: lines.append(f" hash: {asset.digest}") lines.append(f" description: {yaml_scalar(asset.description)}") # Anything under an unrecognised top-level directory still deserves a mention. others = [a for a in assets if a.directory not in {d for d, _ in ASSET_DIRS}] if others: lines.append("") lines.append("unclassified:") for asset in others: lines.append(f" {yaml_key(asset.path)}:") lines.append(f" size: {asset.size}") if asset.digest: lines.append(f" hash: {asset.digest}") lines.append(f" description: {yaml_scalar(asset.description)}") lines.append("...") return "\n".join(lines) + "\n" def render_markdown(root: str, assets: list[Asset], *, title: str | None, by_dir: bool) -> str: total = sum(a.size for a in assets) heading = title or os.path.basename(os.path.abspath(root)) or "Untracked assets" lines = [ f"# Bill of materials — {heading}", "", f"{len(assets)} file(s), {human_size(total)}. Generated by `scripts/bom_gen.py`.", "", ] groups: list[tuple[str, list[Asset]]] = [] if by_dir: for dirname, _role in ASSET_DIRS: grouping = [a for a in assets if a.directory == dirname] if grouping: groups.append((dirname, grouping)) others = [a for a in assets if a.directory not in {d for d, _ in ASSET_DIRS}] if others: groups.append(("unclassified", others)) else: groups.append(("files", assets)) for group_name, grouping in groups: lines.append(f"## {group_name}") lines.append("") # Only carry a hash column when hashes were asked for, so the common # inventory stays narrow enough to read. with_hash = any(asset.digest for asset in grouping) rows = [ ( f"`{asset.path}`", human_size(asset.size), *( (f"`{asset.digest}`",) if with_hash else () ), asset.description or f"_{_DESCRIPTION_PLACEHOLDER}_", ) for asset in grouping ] headers = ("file", "size", "hash", "description") if with_hash else ("file", "size", "description") lines.append(text_table(headers, rows)) lines.append("") return "\n".join(lines).rstrip() + "\n" def validate_yaml(document: str, report: Report) -> None: """If PyYAML is importable, use it to confirm the emitted document parses.""" try: import yaml # type: ignore[import-not-found] except ImportError: return try: yaml.safe_load(document) except Exception as exc: # noqa: BLE001 - report whatever PyYAML says report.warn(f"PyYAML could not parse the generated document: {exc}") # --------------------------------------------------------------------------- # # Interactive prompts # --------------------------------------------------------------------------- # def prompt_for_descriptions(assets: list[Asset], report: Report) -> None: """Ask for a description per asset. EOF ends prompting and keeps what we have.""" if not sys.stdin.isatty(): report.warn( "stdin is not a terminal — skipping prompts; " "descriptions left blank (use --descriptions FILE to supply them)" ) return total = len(assets) report.info(f"Describing {total} file(s). Blank line accepts the current value.\n") report.info("Press Ctrl-C or Ctrl-D to stop early; the BOM is still written.\n") try: for i, asset in enumerate(assets, start=1): report.info(f"[{i}/{total}] {asset.path} ({human_size(asset.size)})") try: answer = input(" description: ").strip() except EOFError: report.info("") report.warn("end of input — remaining descriptions left blank") return if answer: asset.description = answer except KeyboardInterrupt: report.info("") report.warn("interrupted — writing the BOM with the descriptions gathered so far") # --------------------------------------------------------------------------- # # CLI # --------------------------------------------------------------------------- # def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="bom_gen", description="Generate a bill of materials for a song's untracked binary assets.", ) add_common_arguments(parser) parser.add_argument( "root", nargs="?", default=".", help="directory holding audio/, images/, fonts/, videos/ (default: .)", ) parser.add_argument( "-o", "--output", help="output path (default: BOM.yaml or BOM.md in the root directory)", ) parser.add_argument( "--format", default="yaml", choices=("yaml", "md"), help="output format (default: yaml)", ) parser.add_argument( "--descriptions", metavar="FILE", help="read 'path = description' lines from FILE instead of prompting", ) parser.add_argument( "--no-prompt", action="store_true", help="never prompt; write the inventory with blank descriptions", ) parser.add_argument( "--flat", action="store_true", help="list files without grouping them by asset directory", ) parser.add_argument( "--hash", dest="hashes", action="store_true", help="record a short content hash per file (reads every asset; slower)", ) parser.add_argument( "--title", help="song title, recorded in the output header", ) parser.add_argument( "--describe-one", metavar="PATH", help="update the description of a single file in an existing BOM", ) return parser @main_wrapper def main() -> int: args = build_parser().parse_args() report = Report(quiet=args.quiet) root = os.path.abspath(args.root) if not os.path.isdir(root): raise Die(f"not a directory: {args.root}") if args.describe_one: return describe_one(root, args, report) assets = walk_assets(root, hashes=args.hashes) if not assets: expected = ", ".join(f"{name}/" for name, _ in ASSET_DIRS) report.warn(f"no assets found under {root} (looked in: {expected})") return 1 if args.descriptions: mapping = load_descriptions(args.descriptions) applied = 0 for asset in assets: if asset.path in mapping: asset.description = mapping[asset.path] applied += 1 missing = [a.path for a in assets if not a.description] report.verbose(f"applied {applied} description(s) from {args.descriptions}") if missing: report.warn(f"{len(missing)} file(s) have no description in {args.descriptions}") elif not args.no_prompt: prompt_for_descriptions(assets, report) renderer = render_markdown if args.format == "md" else render_yaml document = renderer(root, assets, title=args.title, by_dir=not args.flat) if args.format == "yaml": validate_yaml(document, report) default_name = "BOM.md" if args.format == "md" else "BOM.yaml" output = args.output or os.path.join(root, default_name) if args.dry_run: report.info(document) report.info(f"[dry-run] would write {output}") return 0 with open(output, "w", encoding="utf-8", newline="\n") as handle: handle.write(document) total = sum(a.size for a in assets) described = sum(1 for a in assets if a.description) report.ok(f"{output} — {len(assets)} file(s), {human_size(total)}, {described} described") if described < len(assets): report.info(f" {len(assets) - described} file(s) still need a description") return 0 def describe_one(root: str, args: argparse.Namespace, report: Report) -> int: """Set the description for a single path inside an existing BOM.""" target = args.describe_one.lstrip("./") if not os.path.isfile(os.path.join(root, target)): raise Die(f"no such asset: {target}") output = args.output or os.path.join(root, "BOM.yaml" if args.format == "yaml" else "BOM.md") description = args.title or "" # reused as the description payload for this mode if not description: if not sys.stdin.isatty(): raise Die("--describe-one needs a description; pass --title \"...\"") description = input(f"description for {target}: ").strip() if not os.path.isfile(output): raise Die(f"no existing BOM at {output}; run without --describe-one first") with open(output, "r", encoding="utf-8") as handle: content = handle.read() pattern = re.compile( rf"(^[ \t]*{re.escape(target)}:[ \t]*\n" rf"(?:[ \t]+\w+:.*\n)*?" rf"[ \t]+description:[ \t]*)" rf".*$", re.MULTILINE, ) updated, count = pattern.subn(rf"\g<1>{yaml_scalar(description)}", content) if count == 0: raise Die(f"could not find an entry for {target} in {output}") if args.dry_run: report.info(f"[dry-run] would set {target} description in {output}") return 0 with open(output, "w", encoding="utf-8", newline="\n") as handle: handle.write(updated) report.ok(f"{target} described in {output}") return 0 if __name__ == "__main__": raise SystemExit(main())