diff --git a/scripts/README.md b/scripts/README.md index 1a95253..482c5ad 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -195,11 +195,23 @@ It writes `BOM.yaml` listing every untracked file under `audio/`, `images/`, used for clean output; otherwise a small built-in writer produces the same schema. Neither is required. +Add `--hash` to record a short content hash per file: + +```sh +python3 scripts/tools.py bom --dir . --hash +``` + +That is how you answer "has this asset changed since the BOM was written?" — +sizes stay identical when you re-render a take at the same length, but the hash +will not. Hashing reads every asset in full, so it is off by default; when you +only want the inventory, leave it out. + --- ## Conventions -- `master` is the protected template. Do per-song work on `/master`, and - features on `/`. See `README.md`. +- `master` is the shared template. It is intended to freeze once the workspace is + finalised; until then, tooling work can land on it directly. Per-song work goes + on `/master`, and features on `/`. See `README.md`. - Binary assets stay out of git; the BOM is how you know what was used. - Keep new tools in `scripts/`, stdlib only, with a `--dry-run` and `--help`. diff --git a/scripts/bom_gen.py b/scripts/bom_gen.py index 753620f..d1ff2fd 100644 --- a/scripts/bom_gen.py +++ b/scripts/bom_gen.py @@ -20,6 +20,7 @@ Usage:: from __future__ import annotations import argparse +import hashlib import os import re import sys @@ -57,7 +58,7 @@ ASSET_DIRS: tuple[tuple[str, str], ...] = ( #: File extensions that are worth flagging as (probably) not-yet-described. _DESCRIPTION_PLACEHOLDER = "" -_BOM_MARKER = "; generated by scripts/bom_gen.py" +_BOM_MARKER = "# generated by scripts/bom_gen.py" # --------------------------------------------------------------------------- # # YAML emission (hand-rolled, stdlib only) @@ -100,6 +101,7 @@ class Asset: path: str # relative to the BOM root, POSIX separators size: int description: str = "" + digest: str = "" # short content hash, empty when --hash was not requested extra: dict[str, str] = field(default_factory=dict) @property @@ -107,8 +109,32 @@ class Asset: return os.path.dirname(self.path).split("/")[0] or "." -def walk_assets(root: str, dirs: tuple[tuple[str, str], ...] = ASSET_DIRS) -> list[Asset]: - """Collect files under each asset directory, sorted by path.""" +def file_digest(full: str, *, length: int = 12, chunk: int = 1 << 20) -> str: + """Short BLAKE2b content hash, streamed so large assets do not land in RAM. + + Assets here are audio and video files that can run to hundreds of megabytes, + so this reads in chunks rather than slurping the whole file. BLAKE2b is in + the standard library and is faster than md5 with no collision worries. + ``length`` bytes of digest = 3 * length hex characters. + """ + h = hashlib.blake2b(digest_size=length) + with open(full, "rb") as fh: + while bytes_read := fh.read(chunk): + h.update(bytes_read) + return h.hexdigest() + + +def walk_assets( + root: str, + dirs: tuple[tuple[str, str], ...] = ASSET_DIRS, + *, + hashes: bool = False, +) -> list[Asset]: + """Collect files under each asset directory, sorted by path. + + Hashes are opt-in: they cost a full read of every asset, which is wasted + work when you only want the inventory. + """ assets: list[Asset] = [] for name, _role in dirs: @@ -125,7 +151,13 @@ def walk_assets(root: str, dirs: tuple[tuple[str, str], ...] = ASSET_DIRS) -> li if not os.path.isfile(full): continue rel = os.path.relpath(full, root).replace(os.sep, "/") - assets.append(Asset(path=rel, size=os.path.getsize(full))) + assets.append( + Asset( + path=rel, + size=os.path.getsize(full), + digest=file_digest(full) if hashes else "", + ) + ) return assets @@ -190,6 +222,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo for asset in grouping: lines.append(f" {yaml_key(asset.path)}:") lines.append(f" size: {asset.size}") + if asset.digest: + lines.append(f" hash: {asset.digest}") lines.append(f" description: {yaml_scalar(asset.description)}") lines.append("") else: @@ -197,6 +231,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo for asset in assets: lines.append(f" {yaml_key(asset.path)}:") lines.append(f" size: {asset.size}") + if asset.digest: + lines.append(f" hash: {asset.digest}") lines.append(f" description: {yaml_scalar(asset.description)}") # Anything under an unrecognised top-level directory still deserves a mention. @@ -207,6 +243,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo for asset in others: lines.append(f" {yaml_key(asset.path)}:") lines.append(f" size: {asset.size}") + if asset.digest: + lines.append(f" hash: {asset.digest}") lines.append(f" description: {yaml_scalar(asset.description)}") lines.append("...") @@ -240,15 +278,20 @@ def render_markdown(root: str, assets: list[Asset], *, title: str | None, by_dir for group_name, grouping in groups: lines.append(f"## {group_name}") lines.append("") + # Only carry a hash column when hashes were asked for, so the common + # inventory stays narrow enough to read. + with_hash = any(asset.digest for asset in grouping) rows = [ ( f"`{asset.path}`", human_size(asset.size), + *( (f"`{asset.digest}`",) if with_hash else () ), asset.description or f"_{_DESCRIPTION_PLACEHOLDER}_", ) for asset in grouping ] - lines.append(text_table(("file", "size", "description"), rows)) + headers = ("file", "size", "hash", "description") if with_hash else ("file", "size", "description") + lines.append(text_table(headers, rows)) lines.append("") return "\n".join(lines).rstrip() + "\n" @@ -345,6 +388,12 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="list files without grouping them by asset directory", ) + parser.add_argument( + "--hash", + dest="hashes", + action="store_true", + help="record a short content hash per file (reads every asset; slower)", + ) parser.add_argument( "--title", help="song title, recorded in the output header", @@ -370,7 +419,7 @@ def main() -> int: if args.describe_one: return describe_one(root, args, report) - assets = walk_assets(root) + assets = walk_assets(root, hashes=args.hashes) if not assets: expected = ", ".join(f"{name}/" for name, _ in ASSET_DIRS)