diff --git a/.gitignore b/.gitignore index a5983c8..46e1872 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ cookies.txt *.mkv *.mp4 *.ts +__pycache__/ diff --git a/record_live.py b/record_live.py index 9013526..1e2607d 100644 --- a/record_live.py +++ b/record_live.py @@ -13,6 +13,16 @@ outlive kickoff by months. # start now, keep looking for up to 45 min, then record 150 min python record_live.py --max-wait 45 --duration 150 + # take both feeds at once, or pin one when you know which you hold + python record_live.py --feed both + python record_live.py --entry 0_xm5vrrw6 + +Two different choices, often confused. **--feed** picks which *entry* to record +when the page exposes more than one for the same match, typically an audio feed +and a video feed. **--prefer** picks a rendition *inside* one entry's master. +Getting --feed wrong cannot be rescued by --prefer, and it fails quietly: you +get a complete, playable, perfectly healthy recording of the wrong thing. + # or wait until a wall-clock time first (03:15 before an 03:45 kickoff) python record_live.py --at "2026-09-02 03:15" --max-wait 60 @@ -70,9 +80,13 @@ TOKEN_URL = ("https://www.bristolrovers.co.uk/oauth/browser-token" API_KEY = "50ivbQ3IXpF2mwT4tPAJ2V6cVm4CGRT2YKmDK5v6" -def mint_master(entry_id): +def mint_master(entry_id, quiet=False): """Mint a fresh master playlist URL in plain Python. No browser. + Returns (master_url, rendition_kinds). The master token lives ~25 seconds, + so mint immediately before you use the URL -- classify with one call, then + mint again for the recording. + Handing yt-dlp the *media* playlist (hdntl, path token) fails: Python clients get 403 on that URL form, and yt-dlp fetches the playlist with urllib before ffmpeg ever runs. The *master* (hdnts, query token) is fetched @@ -96,7 +110,7 @@ def mint_master(entry_id): m = re.search(r"(eyJ[\w-]+\.eyJ[\w-]+\.[\w-]+)", body) if not m: log(" mint: no JWT in browser-token response (session expired?)") - return None + return None, [] jwt = m.group(1) req = urllib.request.Request( @@ -107,14 +121,17 @@ def mint_master(entry_id): data = json.loads(urllib.request.urlopen(req, timeout=20) .read().decode("utf-8", "replace")) hls = (data.get("media") or {}).get("hls") or None - if hls: - log_renditions(hls) - return hls + kinds = log_renditions(hls, quiet=quiet) if hls else [] + return hls, kinds -def log_renditions(master_url): - """Record what the master offers. We have never seen one carrying both - video and audio, so log it every time until we know how that looks.""" +def log_renditions(master_url, quiet=False): + """Log what the master offers, and return the kind of each rendition. + + The return value is what feed_kind() classifies an entry on. Codecs are the + only trustworthy signal: entry ids look alike, and the same id comes back + week after week for the same feed. + """ import urllib.request try: req = urllib.request.Request(master_url, headers={"User-Agent": UA, @@ -122,31 +139,51 @@ def log_renditions(master_url): body = urllib.request.urlopen(req, timeout=15).read().decode("utf-8", "replace") except Exception as exc: log(f" (could not read master for rendition list: {exc})") - return + return [] PREFIX = "#EXT-X-STREAM-INF:" streams = [ln[len(PREFIX):] for ln in body.splitlines() if ln.startswith(PREFIX)] - log(f" master offers {len(streams)} rendition(s):") + kinds = [] + if not quiet: + log(f" master offers {len(streams)} rendition(s):") for st in streams: bw = re.search(r"BANDWIDTH=(\d+)", st) res = re.search(r"RESOLUTION=([\dx]+)", st) cod = re.search(r'CODECS="([^"]+)"', st) codecs = cod.group(1) if cod else "?" kind = "audio-only" if ("avc" not in codecs and "hvc" not in codecs) else "video" - log(f" {kind:11s} {int(bw.group(1))//1000 if bw else '?':>6} kbps " - f"{res.group(1) if res else '-':>10} {codecs}") + kinds.append(kind) + if not quiet: + log(f" {kind:11s} {int(bw.group(1))//1000 if bw else '?':>6} kbps " + f"{res.group(1) if res else '-':>10} {codecs}") + return kinds -def find_manifest(timeout_s=90): - """Load the live page headlessly and return the entry id, or None. +def feed_kind(kinds): + """'video' if the master carries any video rendition, else 'audio'.""" + if not kinds: + return "unknown" + return "video" if any(k == "video" for k in kinds) else "audio" + + +def find_manifest(timeout_s=90, settle_s=15): + """Load the live page headlessly and return every entry id it exposes. The browser is needed only for this: the entry id appears at runtime, and the static HTML carries none when nothing is live. Everything after this is plain HTTP. + + Returns a list, not one id, because the page can expose more than one feed + for the same match -- an audio entry and a video entry. Which manifest + loads first is a race, and it has been observed to come out differently on + the same night. Returning the first one silently records the wrong feed. + So keep watching for `settle_s` after the first hit, and let the caller + choose. """ from playwright.sync_api import sync_playwright - found = {"master": None, "media": None} + entries = [] # discovery order, de-duplicated + first_seen = [None] with sync_playwright() as p: browser = p.firefox.launch(headless=True) @@ -159,10 +196,14 @@ def find_manifest(timeout_s=90): u = resp.url if ".m3u8" not in u.split("?")[0].lower(): return - if "hdntl=" in u and not found["media"]: - found["media"] = u - elif "hdnts=" in u and not found["master"]: - found["master"] = u + if "hdntl=" not in u and "hdnts=" not in u: + return + m = re.search(r"/(\d+_[a-z0-9]+)/", u, re.I) + if not m or m.group(1) in entries: + return + entries.append(m.group(1)) + if first_seen[0] is None: + first_seen[0] = time.time() page.on("response", on_response) try: @@ -176,7 +217,11 @@ def find_manifest(timeout_s=90): # because a stream that has only just gone live may need the nudge. deadline = time.time() + timeout_s clicked = False - while time.time() < deadline and not found["media"]: + while time.time() < deadline: + # Stop once the page has settled: a feed the player never requests + # will not appear no matter how long we wait. + if first_seen[0] and time.time() - first_seen[0] > settle_s: + break if not clicked: for sel in (".bmpui-ui-hugeplaybacktogglebutton", ".bmpui-ui-playbacktogglebutton", @@ -196,16 +241,13 @@ def find_manifest(timeout_s=90): browser.close() - seen_url = found["media"] or found["master"] - if not seen_url: - return None - m = re.search(r"/(\d+_[a-z0-9]+)/", seen_url, re.I) - return m.group(1) if m else None + return entries -def record(url, duration_min, prefer='best'): +def record(url, duration_min, prefer='best', tag=None): stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out = os.path.join(RECORDINGS, f"brfc-{stamp}.ts") + name = f"brfc-{stamp}-{tag}.ts" if tag else f"brfc-{stamp}.ts" + out = os.path.join(RECORDINGS, name) os.makedirs(RECORDINGS, exist_ok=True) logfile = os.path.join(LOGS, f"yt-dlp-{stamp}.log") cmd = [ @@ -239,6 +281,99 @@ def record(url, duration_min, prefer='best'): return subprocess.Popen(cmd, stdout=fh, stderr=subprocess.STDOUT, cwd=ROOT) +def classify_entries(entries): + """Label each discovered entry video or audio, by minting its master once. + + Returns [(entry_id, kind)] in discovery order, skipping anything that will + not mint. The master minted here is thrown away -- its token lives ~25 + seconds, so the recording mints its own. + """ + out = [] + for entry in entries: + try: + url, kinds = mint_master(entry) + except Exception as exc: + log(f" {entry}: mint failed: {type(exc).__name__}: {exc}") + continue + if not url: + continue + kind = feed_kind(kinds) + log(f" {entry} is a {kind} feed") + out.append((entry, kind)) + return out + + +def choose_feeds(chosen, want): + """Order classified entries by preference. 'both' and 'any' take them all. + + A preference is not a filter: the non-preferred feeds stay on the list as + fallbacks, because a feed you hold no pass for still resolves a master and + only fails later, on the segments. + """ + if want in ("both", "any"): + return chosen + first = [c for c in chosen if c[1] == want] + rest = [c for c in chosen if c[1] != want] + if not first and rest: + log(f" no {want} feed on offer -- falling back to " + f"{', '.join(k for _, k in rest)}") + return first + rest + + +def run_recorders(targets, args): + """Record, and report whether a recording actually ran. + + 'both' starts every feed at once: concurrency is not limited and a second + copy is cheap insurance. Otherwise take the first choice and fall through + to the next only when the downloader dies immediately -- which is what an + unentitled feed looks like, the master resolving and the segments 403ing. + """ + def start(entry, kind): + try: + url, _ = mint_master(entry, quiet=True) + except Exception as exc: + log(f" {entry}: mint failed: {type(exc).__name__}: {exc}") + return None + if not url: + log(f" {entry}: could not re-mint a master; skipping") + return None + proc = record(url, args.duration, args.prefer, tag=kind) + log(f" {entry} ({kind}): downloader pid {proc.pid}") + return proc, time.time() + + def finish(entry, kind, proc, t0): + proc.wait() + ran = time.time() - t0 + log(f" {entry} ({kind}): rc={proc.returncode} after {ran:.0f}s") + # A master outlives its stream, so minting one proves nothing about + # whether the match is up. Only a download that actually ran does. + if proc.returncode == 0 or ran > 120: + return True + log(" died immediately -- not live yet, or no pass for this feed") + return False + + if args.feed == "both": + started = [] + for entry, kind in targets: + s = start(entry, kind) + if s is not None: + started.append((entry, kind) + s) + # Every one must be waited on, so collect results before reducing -- + # any() would short-circuit and orphan a running downloader. + results = [finish(*row) for row in started] + return any(results) + + # One feed at a time, falling through on immediate death so an unentitled + # first choice does not cost the match. + for entry, kind in targets: + s = start(entry, kind) + if s is None: + continue + if finish(entry, kind, *s): + return True + return False + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--at", help='wall-clock start, e.g. "2026-09-02 03:15"') @@ -250,6 +385,15 @@ def main(): ap.add_argument("--prefer", choices=("best", "audio", "video"), default="best", help="which rendition to take when the master offers both; " "'best' is yt-dlp's default and prefers video") + ap.add_argument("--entry", help="skip discovery and record this entry id, " + "bypassing --feed entirely") + ap.add_argument("--feed", choices=("video", "audio", "any", "both"), + default="video", + help="which FEED to record when the page exposes more than " + "one entry. 'video' prefers a video feed and falls " + "back to audio; 'both' records every feed in parallel. " + "Distinct from --prefer, which picks a rendition " + "inside one feed and cannot rescue a wrong feed.") args = ap.parse_args() if args.at: @@ -263,42 +407,37 @@ def main(): attempt = 0 while time.time() < deadline: attempt += 1 - log(f"attempt {attempt}: looking for a live stream") - try: - entry_id = find_manifest() - except Exception as exc: - log(f" error: {type(exc).__name__}: {exc}") - entry_id = None - if entry_id: - log(f" live entry: {entry_id}") + if args.entry: + entries = [args.entry] + log(f"attempt {attempt}: pinned entry {args.entry}") + else: + log(f"attempt {attempt}: looking for a live stream") try: - url = mint_master(entry_id) + entries = find_manifest() except Exception as exc: - log(f" mint failed: {type(exc).__name__}: {exc}") - url = None - if not url: - log(" could not mint a manifest; retrying") - time.sleep(args.poll) - continue - log(" minted fresh master playlist") - proc = record(url, args.duration, args.prefer) - log(f" downloader pid {proc.pid}; waiting for it to finish") - t0 = time.time() - proc.wait() - ran = time.time() - t0 - log(f" downloader exited rc={proc.returncode} after {ran:.0f}s") + log(f" error: {type(exc).__name__}: {exc}") + entries = [] - # A master playlist outlives its stream: after a match ends the - # master is still served while its variants 404. So "found a - # master" does not mean "stream is live" -- only a download that - # actually ran does. Anything that dies immediately means the - # stream is not up yet, so keep waiting rather than giving up. - if proc.returncode == 0 or ran > 120: - return 0 - log(" died immediately -- stream not live yet, will retry") + if not entries: + log(f" nothing live yet; retrying in {args.poll}s") time.sleep(args.poll) continue - log(f" nothing live yet; retrying in {args.poll}s") + + log(f" live entries: {', '.join(entries)}") + chosen = classify_entries(entries) + if not chosen: + log(" could not mint a manifest for any entry; retrying") + time.sleep(args.poll) + continue + + targets = choose_feeds(chosen, args.feed) + if not targets: + log(f" no {args.feed} feed among {[k for _, k in chosen]}; retrying") + time.sleep(args.poll) + continue + + if run_recorders(targets, args) or time.time() >= deadline: + return 0 time.sleep(args.poll) log("gave up: no live stream appeared within the wait window")