feat(bom): add --hash, and fix the BOM marker breaking YAML
`feat/bom-generator` carried one commit that never landed on master: file
content hashes in the BOM ("v2 with hashes"). The Python rewrite dropped that
capability, so port it forward rather than losing it when the branch goes.
- `--hash` records a short BLAKE2b digest per file. Opt-in, because it reads
every asset in full and that is wasted work for a plain inventory.
- Hashes are streamed in chunks with `hashlib.blake2b`, not `md5` via haveged
imports, and not whole-file reads — these assets run to hundreds of megabytes.
- YAML gains `hash:` after `size:`; Markdown gains a hash column only when
hashes were requested, so the common table stays readable.
Fix a pre-existing bug found while testing: the BOM marker was written as
`; generated by ...`, but `;` is not a YAML comment, so every generated BOM
failed to parse with "mapping values are not allowed here". Now `#`. Verified by
re-parsing output from both the PyYAML path and the built-in fallback writer.
Sizes alone cannot tell you whether an asset changed; a same-length re-render
keeps the byte count identical. That is the gap the hash closes.
This commit is contained in:
+14
-2
@@ -195,11 +195,23 @@ It writes `BOM.yaml` listing every untracked file under `audio/`, `images/`,
|
|||||||
used for clean output; otherwise a small built-in writer produces the same
|
used for clean output; otherwise a small built-in writer produces the same
|
||||||
schema. Neither is required.
|
schema. Neither is required.
|
||||||
|
|
||||||
|
Add `--hash` to record a short content hash per file:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 scripts/tools.py bom --dir . --hash
|
||||||
|
```
|
||||||
|
|
||||||
|
That is how you answer "has this asset changed since the BOM was written?" —
|
||||||
|
sizes stay identical when you re-render a take at the same length, but the hash
|
||||||
|
will not. Hashing reads every asset in full, so it is off by default; when you
|
||||||
|
only want the inventory, leave it out.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Conventions
|
## Conventions
|
||||||
|
|
||||||
- `master` is the protected template. Do per-song work on `<song>/master`, and
|
- `master` is the shared template. It is intended to freeze once the workspace is
|
||||||
features on `<song>/<feature>`. See `README.md`.
|
finalised; until then, tooling work can land on it directly. Per-song work goes
|
||||||
|
on `<song>/master`, and features on `<song>/<feature>`. See `README.md`.
|
||||||
- Binary assets stay out of git; the BOM is how you know what was used.
|
- Binary assets stay out of git; the BOM is how you know what was used.
|
||||||
- Keep new tools in `scripts/`, stdlib only, with a `--dry-run` and `--help`.
|
- Keep new tools in `scripts/`, stdlib only, with a `--dry-run` and `--help`.
|
||||||
|
|||||||
+55
-6
@@ -20,6 +20,7 @@ Usage::
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import hashlib
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
@@ -57,7 +58,7 @@ ASSET_DIRS: tuple[tuple[str, str], ...] = (
|
|||||||
#: File extensions that are worth flagging as (probably) not-yet-described.
|
#: File extensions that are worth flagging as (probably) not-yet-described.
|
||||||
_DESCRIPTION_PLACEHOLDER = "<describe this file>"
|
_DESCRIPTION_PLACEHOLDER = "<describe this file>"
|
||||||
|
|
||||||
_BOM_MARKER = "; generated by scripts/bom_gen.py"
|
_BOM_MARKER = "# generated by scripts/bom_gen.py"
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
# YAML emission (hand-rolled, stdlib only)
|
# YAML emission (hand-rolled, stdlib only)
|
||||||
@@ -100,6 +101,7 @@ class Asset:
|
|||||||
path: str # relative to the BOM root, POSIX separators
|
path: str # relative to the BOM root, POSIX separators
|
||||||
size: int
|
size: int
|
||||||
description: str = ""
|
description: str = ""
|
||||||
|
digest: str = "" # short content hash, empty when --hash was not requested
|
||||||
extra: dict[str, str] = field(default_factory=dict)
|
extra: dict[str, str] = field(default_factory=dict)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -107,8 +109,32 @@ class Asset:
|
|||||||
return os.path.dirname(self.path).split("/")[0] or "."
|
return os.path.dirname(self.path).split("/")[0] or "."
|
||||||
|
|
||||||
|
|
||||||
def walk_assets(root: str, dirs: tuple[tuple[str, str], ...] = ASSET_DIRS) -> list[Asset]:
|
def file_digest(full: str, *, length: int = 12, chunk: int = 1 << 20) -> str:
|
||||||
"""Collect files under each asset directory, sorted by path."""
|
"""Short BLAKE2b content hash, streamed so large assets do not land in RAM.
|
||||||
|
|
||||||
|
Assets here are audio and video files that can run to hundreds of megabytes,
|
||||||
|
so this reads in chunks rather than slurping the whole file. BLAKE2b is in
|
||||||
|
the standard library and is faster than md5 with no collision worries.
|
||||||
|
``length`` bytes of digest = 3 * length hex characters.
|
||||||
|
"""
|
||||||
|
h = hashlib.blake2b(digest_size=length)
|
||||||
|
with open(full, "rb") as fh:
|
||||||
|
while bytes_read := fh.read(chunk):
|
||||||
|
h.update(bytes_read)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def walk_assets(
|
||||||
|
root: str,
|
||||||
|
dirs: tuple[tuple[str, str], ...] = ASSET_DIRS,
|
||||||
|
*,
|
||||||
|
hashes: bool = False,
|
||||||
|
) -> list[Asset]:
|
||||||
|
"""Collect files under each asset directory, sorted by path.
|
||||||
|
|
||||||
|
Hashes are opt-in: they cost a full read of every asset, which is wasted
|
||||||
|
work when you only want the inventory.
|
||||||
|
"""
|
||||||
assets: list[Asset] = []
|
assets: list[Asset] = []
|
||||||
|
|
||||||
for name, _role in dirs:
|
for name, _role in dirs:
|
||||||
@@ -125,7 +151,13 @@ def walk_assets(root: str, dirs: tuple[tuple[str, str], ...] = ASSET_DIRS) -> li
|
|||||||
if not os.path.isfile(full):
|
if not os.path.isfile(full):
|
||||||
continue
|
continue
|
||||||
rel = os.path.relpath(full, root).replace(os.sep, "/")
|
rel = os.path.relpath(full, root).replace(os.sep, "/")
|
||||||
assets.append(Asset(path=rel, size=os.path.getsize(full)))
|
assets.append(
|
||||||
|
Asset(
|
||||||
|
path=rel,
|
||||||
|
size=os.path.getsize(full),
|
||||||
|
digest=file_digest(full) if hashes else "",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
return assets
|
return assets
|
||||||
|
|
||||||
@@ -190,6 +222,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo
|
|||||||
for asset in grouping:
|
for asset in grouping:
|
||||||
lines.append(f" {yaml_key(asset.path)}:")
|
lines.append(f" {yaml_key(asset.path)}:")
|
||||||
lines.append(f" size: {asset.size}")
|
lines.append(f" size: {asset.size}")
|
||||||
|
if asset.digest:
|
||||||
|
lines.append(f" hash: {asset.digest}")
|
||||||
lines.append(f" description: {yaml_scalar(asset.description)}")
|
lines.append(f" description: {yaml_scalar(asset.description)}")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
else:
|
else:
|
||||||
@@ -197,6 +231,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo
|
|||||||
for asset in assets:
|
for asset in assets:
|
||||||
lines.append(f" {yaml_key(asset.path)}:")
|
lines.append(f" {yaml_key(asset.path)}:")
|
||||||
lines.append(f" size: {asset.size}")
|
lines.append(f" size: {asset.size}")
|
||||||
|
if asset.digest:
|
||||||
|
lines.append(f" hash: {asset.digest}")
|
||||||
lines.append(f" description: {yaml_scalar(asset.description)}")
|
lines.append(f" description: {yaml_scalar(asset.description)}")
|
||||||
|
|
||||||
# Anything under an unrecognised top-level directory still deserves a mention.
|
# Anything under an unrecognised top-level directory still deserves a mention.
|
||||||
@@ -207,6 +243,8 @@ def render_yaml(root: str, assets: list[Asset], *, title: str | None, by_dir: bo
|
|||||||
for asset in others:
|
for asset in others:
|
||||||
lines.append(f" {yaml_key(asset.path)}:")
|
lines.append(f" {yaml_key(asset.path)}:")
|
||||||
lines.append(f" size: {asset.size}")
|
lines.append(f" size: {asset.size}")
|
||||||
|
if asset.digest:
|
||||||
|
lines.append(f" hash: {asset.digest}")
|
||||||
lines.append(f" description: {yaml_scalar(asset.description)}")
|
lines.append(f" description: {yaml_scalar(asset.description)}")
|
||||||
|
|
||||||
lines.append("...")
|
lines.append("...")
|
||||||
@@ -240,15 +278,20 @@ def render_markdown(root: str, assets: list[Asset], *, title: str | None, by_dir
|
|||||||
for group_name, grouping in groups:
|
for group_name, grouping in groups:
|
||||||
lines.append(f"## {group_name}")
|
lines.append(f"## {group_name}")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
# Only carry a hash column when hashes were asked for, so the common
|
||||||
|
# inventory stays narrow enough to read.
|
||||||
|
with_hash = any(asset.digest for asset in grouping)
|
||||||
rows = [
|
rows = [
|
||||||
(
|
(
|
||||||
f"`{asset.path}`",
|
f"`{asset.path}`",
|
||||||
human_size(asset.size),
|
human_size(asset.size),
|
||||||
|
*( (f"`{asset.digest}`",) if with_hash else () ),
|
||||||
asset.description or f"_{_DESCRIPTION_PLACEHOLDER}_",
|
asset.description or f"_{_DESCRIPTION_PLACEHOLDER}_",
|
||||||
)
|
)
|
||||||
for asset in grouping
|
for asset in grouping
|
||||||
]
|
]
|
||||||
lines.append(text_table(("file", "size", "description"), rows))
|
headers = ("file", "size", "hash", "description") if with_hash else ("file", "size", "description")
|
||||||
|
lines.append(text_table(headers, rows))
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
return "\n".join(lines).rstrip() + "\n"
|
return "\n".join(lines).rstrip() + "\n"
|
||||||
@@ -345,6 +388,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="list files without grouping them by asset directory",
|
help="list files without grouping them by asset directory",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--hash",
|
||||||
|
dest="hashes",
|
||||||
|
action="store_true",
|
||||||
|
help="record a short content hash per file (reads every asset; slower)",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--title",
|
"--title",
|
||||||
help="song title, recorded in the output header",
|
help="song title, recorded in the output header",
|
||||||
@@ -370,7 +419,7 @@ def main() -> int:
|
|||||||
if args.describe_one:
|
if args.describe_one:
|
||||||
return describe_one(root, args, report)
|
return describe_one(root, args, report)
|
||||||
|
|
||||||
assets = walk_assets(root)
|
assets = walk_assets(root, hashes=args.hashes)
|
||||||
|
|
||||||
if not assets:
|
if not assets:
|
||||||
expected = ", ".join(f"{name}/" for name, _ in ASSET_DIRS)
|
expected = ", ".join(f"{name}/" for name, _ in ASSET_DIRS)
|
||||||
|
|||||||
Reference in New Issue
Block a user