From 847fe1e13f0da73040b4904183c090cff1782153 Mon Sep 17 00:00:00 2001 From: Mikkeli Matlock Date: Sat, 26 Sep 2026 22:33:34 +0900 Subject: [PATCH] Choose the feed, not just the rendition The live page can expose two entries for one match, an audio feed and a video feed, and discovery returned whichever manifest loaded first. That race came out differently twice on the same night: at 21:11 the page served the video entry, at 22:30 the audio one. The scheduled run recorded audio against a video pass, and nothing looked wrong -- it fails by producing a complete, playable, healthy recording of the wrong thing. --prefer cannot rescue that; it picks a rendition inside one master, after the entry is already chosen. So add --feed, which chooses between entries: video (default) prefer a video feed, fall back to audio audio the reverse any first discovered, the old behaviour both record every feed in parallel --entry skip discovery and pin one find_manifest now returns every entry it sees and keeps watching 15s past the first hit, rather than returning on it. Each candidate is classified by the codecs in its master, never by entry id -- ids name a feed, but relying on that would bake in an observation that has already been wrong once. A preference is not a filter. Non-preferred feeds stay on the list, because a feed you hold no pass for resolves a master fine and only 403s later on the segments; the recorder falls through to the next candidate when a download dies immediately, so a wrong guess costs seconds rather than the match. Output files are tagged -video / -audio, which matters once --feed both writes two at once. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Fr1fLMnVGNAKH5L2BmhMxn --- .gitignore | 1 + record_live.py | 255 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 198 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index a5983c8..46e1872 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ cookies.txt *.mkv *.mp4 *.ts +__pycache__/ diff --git a/record_live.py b/record_live.py index 9013526..1e2607d 100644 --- a/record_live.py +++ b/record_live.py @@ -13,6 +13,16 @@ outlive kickoff by months. # start now, keep looking for up to 45 min, then record 150 min python record_live.py --max-wait 45 --duration 150 + # take both feeds at once, or pin one when you know which you hold + python record_live.py --feed both + python record_live.py --entry 0_xm5vrrw6 + +Two different choices, often confused. **--feed** picks which *entry* to record +when the page exposes more than one for the same match, typically an audio feed +and a video feed. **--prefer** picks a rendition *inside* one entry's master. +Getting --feed wrong cannot be rescued by --prefer, and it fails quietly: you +get a complete, playable, perfectly healthy recording of the wrong thing. + # or wait until a wall-clock time first (03:15 before an 03:45 kickoff) python record_live.py --at "2026-09-02 03:15" --max-wait 60 @@ -70,9 +80,13 @@ TOKEN_URL = ("https://www.bristolrovers.co.uk/oauth/browser-token" API_KEY = "50ivbQ3IXpF2mwT4tPAJ2V6cVm4CGRT2YKmDK5v6" -def mint_master(entry_id): +def mint_master(entry_id, quiet=False): """Mint a fresh master playlist URL in plain Python. No browser. + Returns (master_url, rendition_kinds). The master token lives ~25 seconds, + so mint immediately before you use the URL -- classify with one call, then + mint again for the recording. + Handing yt-dlp the *media* playlist (hdntl, path token) fails: Python clients get 403 on that URL form, and yt-dlp fetches the playlist with urllib before ffmpeg ever runs. The *master* (hdnts, query token) is fetched @@ -96,7 +110,7 @@ def mint_master(entry_id): m = re.search(r"(eyJ[\w-]+\.eyJ[\w-]+\.[\w-]+)", body) if not m: log(" mint: no JWT in browser-token response (session expired?)") - return None + return None, [] jwt = m.group(1) req = urllib.request.Request( @@ -107,14 +121,17 @@ def mint_master(entry_id): data = json.loads(urllib.request.urlopen(req, timeout=20) .read().decode("utf-8", "replace")) hls = (data.get("media") or {}).get("hls") or None - if hls: - log_renditions(hls) - return hls + kinds = log_renditions(hls, quiet=quiet) if hls else [] + return hls, kinds -def log_renditions(master_url): - """Record what the master offers. We have never seen one carrying both - video and audio, so log it every time until we know how that looks.""" +def log_renditions(master_url, quiet=False): + """Log what the master offers, and return the kind of each rendition. + + The return value is what feed_kind() classifies an entry on. Codecs are the + only trustworthy signal: entry ids look alike, and the same id comes back + week after week for the same feed. + """ import urllib.request try: req = urllib.request.Request(master_url, headers={"User-Agent": UA, @@ -122,31 +139,51 @@ def log_renditions(master_url): body = urllib.request.urlopen(req, timeout=15).read().decode("utf-8", "replace") except Exception as exc: log(f" (could not read master for rendition list: {exc})") - return + return [] PREFIX = "#EXT-X-STREAM-INF:" streams = [ln[len(PREFIX):] for ln in body.splitlines() if ln.startswith(PREFIX)] - log(f" master offers {len(streams)} rendition(s):") + kinds = [] + if not quiet: + log(f" master offers {len(streams)} rendition(s):") for st in streams: bw = re.search(r"BANDWIDTH=(\d+)", st) res = re.search(r"RESOLUTION=([\dx]+)", st) cod = re.search(r'CODECS="([^"]+)"', st) codecs = cod.group(1) if cod else "?" kind = "audio-only" if ("avc" not in codecs and "hvc" not in codecs) else "video" - log(f" {kind:11s} {int(bw.group(1))//1000 if bw else '?':>6} kbps " - f"{res.group(1) if res else '-':>10} {codecs}") + kinds.append(kind) + if not quiet: + log(f" {kind:11s} {int(bw.group(1))//1000 if bw else '?':>6} kbps " + f"{res.group(1) if res else '-':>10} {codecs}") + return kinds -def find_manifest(timeout_s=90): - """Load the live page headlessly and return the entry id, or None. +def feed_kind(kinds): + """'video' if the master carries any video rendition, else 'audio'.""" + if not kinds: + return "unknown" + return "video" if any(k == "video" for k in kinds) else "audio" + + +def find_manifest(timeout_s=90, settle_s=15): + """Load the live page headlessly and return every entry id it exposes. The browser is needed only for this: the entry id appears at runtime, and the static HTML carries none when nothing is live. Everything after this is plain HTTP. + + Returns a list, not one id, because the page can expose more than one feed + for the same match -- an audio entry and a video entry. Which manifest + loads first is a race, and it has been observed to come out differently on + the same night. Returning the first one silently records the wrong feed. + So keep watching for `settle_s` after the first hit, and let the caller + choose. """ from playwright.sync_api import sync_playwright - found = {"master": None, "media": None} + entries = [] # discovery order, de-duplicated + first_seen = [None] with sync_playwright() as p: browser = p.firefox.launch(headless=True) @@ -159,10 +196,14 @@ def find_manifest(timeout_s=90): u = resp.url if ".m3u8" not in u.split("?")[0].lower(): return - if "hdntl=" in u and not found["media"]: - found["media"] = u - elif "hdnts=" in u and not found["master"]: - found["master"] = u + if "hdntl=" not in u and "hdnts=" not in u: + return + m = re.search(r"/(\d+_[a-z0-9]+)/", u, re.I) + if not m or m.group(1) in entries: + return + entries.append(m.group(1)) + if first_seen[0] is None: + first_seen[0] = time.time() page.on("response", on_response) try: @@ -176,7 +217,11 @@ def find_manifest(timeout_s=90): # because a stream that has only just gone live may need the nudge. deadline = time.time() + timeout_s clicked = False - while time.time() < deadline and not found["media"]: + while time.time() < deadline: + # Stop once the page has settled: a feed the player never requests + # will not appear no matter how long we wait. + if first_seen[0] and time.time() - first_seen[0] > settle_s: + break if not clicked: for sel in (".bmpui-ui-hugeplaybacktogglebutton", ".bmpui-ui-playbacktogglebutton", @@ -196,16 +241,13 @@ def find_manifest(timeout_s=90): browser.close() - seen_url = found["media"] or found["master"] - if not seen_url: - return None - m = re.search(r"/(\d+_[a-z0-9]+)/", seen_url, re.I) - return m.group(1) if m else None + return entries -def record(url, duration_min, prefer='best'): +def record(url, duration_min, prefer='best', tag=None): stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out = os.path.join(RECORDINGS, f"brfc-{stamp}.ts") + name = f"brfc-{stamp}-{tag}.ts" if tag else f"brfc-{stamp}.ts" + out = os.path.join(RECORDINGS, name) os.makedirs(RECORDINGS, exist_ok=True) logfile = os.path.join(LOGS, f"yt-dlp-{stamp}.log") cmd = [ @@ -239,6 +281,99 @@ def record(url, duration_min, prefer='best'): return subprocess.Popen(cmd, stdout=fh, stderr=subprocess.STDOUT, cwd=ROOT) +def classify_entries(entries): + """Label each discovered entry video or audio, by minting its master once. + + Returns [(entry_id, kind)] in discovery order, skipping anything that will + not mint. The master minted here is thrown away -- its token lives ~25 + seconds, so the recording mints its own. + """ + out = [] + for entry in entries: + try: + url, kinds = mint_master(entry) + except Exception as exc: + log(f" {entry}: mint failed: {type(exc).__name__}: {exc}") + continue + if not url: + continue + kind = feed_kind(kinds) + log(f" {entry} is a {kind} feed") + out.append((entry, kind)) + return out + + +def choose_feeds(chosen, want): + """Order classified entries by preference. 'both' and 'any' take them all. + + A preference is not a filter: the non-preferred feeds stay on the list as + fallbacks, because a feed you hold no pass for still resolves a master and + only fails later, on the segments. + """ + if want in ("both", "any"): + return chosen + first = [c for c in chosen if c[1] == want] + rest = [c for c in chosen if c[1] != want] + if not first and rest: + log(f" no {want} feed on offer -- falling back to " + f"{', '.join(k for _, k in rest)}") + return first + rest + + +def run_recorders(targets, args): + """Record, and report whether a recording actually ran. + + 'both' starts every feed at once: concurrency is not limited and a second + copy is cheap insurance. Otherwise take the first choice and fall through + to the next only when the downloader dies immediately -- which is what an + unentitled feed looks like, the master resolving and the segments 403ing. + """ + def start(entry, kind): + try: + url, _ = mint_master(entry, quiet=True) + except Exception as exc: + log(f" {entry}: mint failed: {type(exc).__name__}: {exc}") + return None + if not url: + log(f" {entry}: could not re-mint a master; skipping") + return None + proc = record(url, args.duration, args.prefer, tag=kind) + log(f" {entry} ({kind}): downloader pid {proc.pid}") + return proc, time.time() + + def finish(entry, kind, proc, t0): + proc.wait() + ran = time.time() - t0 + log(f" {entry} ({kind}): rc={proc.returncode} after {ran:.0f}s") + # A master outlives its stream, so minting one proves nothing about + # whether the match is up. Only a download that actually ran does. + if proc.returncode == 0 or ran > 120: + return True + log(" died immediately -- not live yet, or no pass for this feed") + return False + + if args.feed == "both": + started = [] + for entry, kind in targets: + s = start(entry, kind) + if s is not None: + started.append((entry, kind) + s) + # Every one must be waited on, so collect results before reducing -- + # any() would short-circuit and orphan a running downloader. + results = [finish(*row) for row in started] + return any(results) + + # One feed at a time, falling through on immediate death so an unentitled + # first choice does not cost the match. + for entry, kind in targets: + s = start(entry, kind) + if s is None: + continue + if finish(entry, kind, *s): + return True + return False + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--at", help='wall-clock start, e.g. "2026-09-02 03:15"') @@ -250,6 +385,15 @@ def main(): ap.add_argument("--prefer", choices=("best", "audio", "video"), default="best", help="which rendition to take when the master offers both; " "'best' is yt-dlp's default and prefers video") + ap.add_argument("--entry", help="skip discovery and record this entry id, " + "bypassing --feed entirely") + ap.add_argument("--feed", choices=("video", "audio", "any", "both"), + default="video", + help="which FEED to record when the page exposes more than " + "one entry. 'video' prefers a video feed and falls " + "back to audio; 'both' records every feed in parallel. " + "Distinct from --prefer, which picks a rendition " + "inside one feed and cannot rescue a wrong feed.") args = ap.parse_args() if args.at: @@ -263,42 +407,37 @@ def main(): attempt = 0 while time.time() < deadline: attempt += 1 - log(f"attempt {attempt}: looking for a live stream") - try: - entry_id = find_manifest() - except Exception as exc: - log(f" error: {type(exc).__name__}: {exc}") - entry_id = None - if entry_id: - log(f" live entry: {entry_id}") + if args.entry: + entries = [args.entry] + log(f"attempt {attempt}: pinned entry {args.entry}") + else: + log(f"attempt {attempt}: looking for a live stream") try: - url = mint_master(entry_id) + entries = find_manifest() except Exception as exc: - log(f" mint failed: {type(exc).__name__}: {exc}") - url = None - if not url: - log(" could not mint a manifest; retrying") - time.sleep(args.poll) - continue - log(" minted fresh master playlist") - proc = record(url, args.duration, args.prefer) - log(f" downloader pid {proc.pid}; waiting for it to finish") - t0 = time.time() - proc.wait() - ran = time.time() - t0 - log(f" downloader exited rc={proc.returncode} after {ran:.0f}s") + log(f" error: {type(exc).__name__}: {exc}") + entries = [] - # A master playlist outlives its stream: after a match ends the - # master is still served while its variants 404. So "found a - # master" does not mean "stream is live" -- only a download that - # actually ran does. Anything that dies immediately means the - # stream is not up yet, so keep waiting rather than giving up. - if proc.returncode == 0 or ran > 120: - return 0 - log(" died immediately -- stream not live yet, will retry") + if not entries: + log(f" nothing live yet; retrying in {args.poll}s") time.sleep(args.poll) continue - log(f" nothing live yet; retrying in {args.poll}s") + + log(f" live entries: {', '.join(entries)}") + chosen = classify_entries(entries) + if not chosen: + log(" could not mint a manifest for any entry; retrying") + time.sleep(args.poll) + continue + + targets = choose_feeds(chosen, args.feed) + if not targets: + log(f" no {args.feed} feed among {[k for _, k in chosen]}; retrying") + time.sleep(args.poll) + continue + + if run_recorders(targets, args) or time.time() >= deadline: + return 0 time.sleep(args.poll) log("gave up: no live stream appeared within the wait window")