mikkeli/ncmr-songs-videos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(bom): add --hash, and fix the BOM marker breaking YAML

Browse Source 
`feat/bom-generator` carried one commit that never landed on master: file
content hashes in the BOM ("v2 with hashes"). The Python rewrite dropped that
capability, so port it forward rather than losing it when the branch goes.

- `--hash` records a short BLAKE2b digest per file. Opt-in, because it reads
  every asset in full and that is wasted work for a plain inventory.
- Hashes are streamed in chunks with `hashlib.blake2b`, not `md5` via haveged
  imports, and not whole-file reads — these assets run to hundreds of megabytes.
- YAML gains `hash:` after `size:`; Markdown gains a hash column only when
  hashes were requested, so the common table stays readable.

Fix a pre-existing bug found while testing: the BOM marker was written as
`; generated by ...`, but `;` is not a YAML comment, so every generated BOM
failed to parse with "mapping values are not allowed here". Now `#`. Verified by
re-parsing output from both the PyYAML path and the built-in fallback writer.

Sizes alone cannot tell you whether an asset changed; a same-length re-render
keeps the byte count identical. That is the gap the hash closes.
This commit is contained in:
Irene Koivisto
2026-09-10 18:37:18 +09:00
parent afe397da3f
commit 2c6e227c82
2 changed files with 69 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/README.md
+14 -2
View File
@@ -195,11 +195,23 @@ It writes `BOM.yaml` listing every untracked file under `audio/`, `images/`,
used for clean output; otherwise a small built-in writer produces the same
schema. Neither is required.
Add `--hash` to record a short content hash per file:
```sh
python3 scripts/tools.py bom --dir . --hash
```
That is how you answer "has this asset changed since the BOM was written?" —
sizes stay identical when you re-render a take at the same length, but the hash
will not. Hashing reads every asset in full, so it is off by default; when you
only want the inventory, leave it out.
---
## Conventions
- `master` is the protected template. Do per-song work on `<song>/master`, and
features on `<song>/<feature>`. See `README.md`.
- `master` is the shared template. It is intended to freeze once the workspace is
finalised; until then, tooling work can land on it directly. Per-song work goes
on `<song>/master`, and features on `<song>/<feature>`. See `README.md`.
- Binary assets stay out of git; the BOM is how you know what was used.
- Keep new tools in `scripts/`, stdlib only, with a `--dry-run` and `--help`.
scripts/bom_gen.py
+55 -6
View File
@@ -20,6 +20,7 @@ Usage::
from __future__ import annotations
import argparse
import hashlib
import os
import re
import sys
@@ -57,7 +58,7 @@ ASSET_DIRS: tuple[tuple[str, str], ...] = (
#: File extensions that are worth flagging as (probably) not-yet-described.
_DESCRIPTION_PLACEHOLDER = "<describe this file>"
_BOM_MARKER = "; generated by scripts/bom_gen.py"
_BOM_MARKER = "# generated by scripts/bom_gen.py"
# --------------------------------------------------------------------------- #
# YAML emission (hand-rolled, stdlib only)
@@ -100,6 +101,7 @@ class Asset:
path: str # relative to the BOM root, POSIX separators
size: int
description: str = ""
digest: str = "" # short content hash, empty when --hash was not requested
extra: dict[str, str] = field(default_factory=dict)
@property
@@ -107,8 +109,32 @@ class Asset:
return os.path.dirname(self.path).split("/")[0] or "."
def walk_assets(root: str, dirs: tuple[tuple[str, str], ...] = ASSET_DIRS) -> list[Asset]:
"""Collect files under each asset directory, sorted by path."""
def file_digest(full: str, *, length: int = 12, chunk: int = 1 << 20) -> str:
"""Short BLAKE2b content hash, streamed so large assets do not land in RAM.
Assets here are audio and video files that can run to hundreds of megabytes,
so this reads in chunks rather than slurping the whole file. BLAKE2b is in
the standard library and is faster than md5 with no collision worries.
``length`` bytes of digest = 3 * length hex characters.
"""
h = hashlib.blake2b(digest_size=length)
with open(full, "rb") as fh:
while bytes_read := fh.read(chunk):
h.update(bytes_read)
return h.hexdigest()
def walk_assets(
root: str,
dirs: tuple[tuple[str, str], ...] = ASSET_DIRS,
*,
hashes: bool = False,
) -> list[Asset]:
"""Collect files under each asset directory, sorted by path.
Hashes are opt-in: they cost a full read of every asset, which is wasted
work when you only want the inventory.
"""
assets: list[Asset] = []
for name, _role in dirs:
@@ -125,7 +151,13 @@ def walk_assets(root: str, dirs: tuple[tuple[str, str], ...] = ASSET_DIRS) -> li
if not os.path.isfile(full):
continue
rel = os.path.relpath(full, root).replace(os.sep, "/")
assets.append(Asset(path=rel, size=os.path.getsize(full)))
assets.append(
Asset(
path=rel,
size=os.path.getsize(full),
digest=file_digest(full) if hashes else "",
)
)
return assets
@@ -190,6 +222,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo
for asset in grouping:
lines.append(f" {yaml_key(asset.path)}:")
lines.append(f" size: {asset.size}")
if asset.digest:
lines.append(f" hash: {asset.digest}")
lines.append(f" description: {yaml_scalar(asset.description)}")
lines.append("")
else:
@@ -197,6 +231,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo
for asset in assets:
lines.append(f" {yaml_key(asset.path)}:")
lines.append(f" size: {asset.size}")
if asset.digest:
lines.append(f" hash: {asset.digest}")
lines.append(f" description: {yaml_scalar(asset.description)}")
# Anything under an unrecognised top-level directory still deserves a mention.
@@ -207,6 +243,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo
for asset in others:
lines.append(f" {yaml_key(asset.path)}:")
lines.append(f" size: {asset.size}")
if asset.digest:
lines.append(f" hash: {asset.digest}")
lines.append(f" description: {yaml_scalar(asset.description)}")
lines.append("...")
@@ -240,15 +278,20 @@ def render_markdown(root: str, assets: list[Asset], *, title: str | None, by_dir
for group_name, grouping in groups:
lines.append(f"## {group_name}")
lines.append("")
# Only carry a hash column when hashes were asked for, so the common
# inventory stays narrow enough to read.
with_hash = any(asset.digest for asset in grouping)
rows = [
(
f"`{asset.path}`",
human_size(asset.size),
*( (f"`{asset.digest}`",) if with_hash else () ),
asset.description or f"_{_DESCRIPTION_PLACEHOLDER}_",
)
for asset in grouping
]
lines.append(text_table(("file", "size", "description"), rows))
headers = ("file", "size", "hash", "description") if with_hash else ("file", "size", "description")
lines.append(text_table(headers, rows))
lines.append("")
return "\n".join(lines).rstrip() + "\n"
@@ -345,6 +388,12 @@ def build_parser() -> argparse.ArgumentParser:
action="store_true",
help="list files without grouping them by asset directory",
)
parser.add_argument(
"--hash",
dest="hashes",
action="store_true",
help="record a short content hash per file (reads every asset; slower)",
)
parser.add_argument(
"--title",
help="song title, recorded in the output header",
@@ -370,7 +419,7 @@ def main() -> int:
if args.describe_one:
return describe_one(root, args, report)
assets = walk_assets(root)
assets = walk_assets(root, hashes=args.hashes)
if not assets:
expected = ", ".join(f"{name}/" for name, _ in ASSET_DIRS)