From f476e852090a309a905efff5150b2749a8e17262 Mon Sep 17 00:00:00 2001 From: Mikkeli Matlock Date: Tue, 1 Sep 2026 21:00:08 +0900 Subject: [PATCH] Add unattended recorder that finds the live stream itself record_live.py runs the whole chain alone: headless browser only to learn the entry id (it appears at runtime; static HTML has none when nothing is live), then plain Python mints a fresh master playlist, then ffmpeg records. No password needed. Firefox session cookies run to late November, well past any kickoff, so the job authenticates from the stored session. Three failures found by testing against a live feed rather than at kickoff: - yt-dlp's native HLS downloader fetches variant playlists with Python and gets 403/404 on path-token URLs. Forced --downloader ffmpeg; this is required, not a preference. - ffmpeg 4.3.1 has no -reconnect_on_network_error. Removed here and in catcher.py, which had the same unsupported flag. - A master playlist outlives its stream: after a match ends the master is still served while its variants 404. So a found master does not mean a live stream, and a downloader that dies immediately means "not up yet" rather than "failed". It now retries until the wait window expires. Untested: the actual segment download for an audio-only stream, because nothing is live right now. Video was verified working on 29 Aug. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Kyt7QHQvLJtQRuiY4wAqKB --- catcher.py | 3 +- record_live.py | 272 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 record_live.py diff --git a/catcher.py b/catcher.py index c7e8cfa..66b6c63 100644 --- a/catcher.py +++ b/catcher.py @@ -68,8 +68,9 @@ def start_recording(key, url, user_agent, page): # yt-dlp's own retry flags do NOT reach the ffmpeg downloader, which is # what actually pulls live HLS. These are ffmpeg's own reconnect knobs. "--downloader-args", + # ffmpeg 4.3.1 has no -reconnect_on_network_error. "ffmpeg_i:-reconnect 1 -reconnect_streamed 1 " - "-reconnect_on_network_error 1 -reconnect_delay_max 30", + "-reconnect_delay_max 30", "--downloader-args", f"ffmpeg_o:-t {DURATION_MIN * 60}", "-o", out, url, diff --git a/record_live.py b/record_live.py new file mode 100644 index 0000000..6c267f4 --- /dev/null +++ b/record_live.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Unattended recorder: find the live stream, then record it. + +Why this needs a headless browser at all: auth and the token mint are plain +HTTP (see discover.py), but the *entry id* for a given match only appears in +the page at runtime -- the static HTML carries no id when nothing is live. So +we render the page headlessly just long enough to learn the manifest URL, then +hand it to ffmpeg and close the browser. + +Nothing here needs your password. It rides the Firefox session cookies, which +outlive kickoff by months. + + # start now, keep looking for up to 45 min, then record 150 min + python record_live.py --max-wait 45 --duration 150 + + # or wait until a wall-clock time first (03:15 before an 03:45 kickoff) + python record_live.py --at "2026-09-02 03:15" --max-wait 60 + +Recording stops on the duration cap. That is deliberate and load-bearing: the +feed does not emit #EXT-X-ENDLIST -- after full time it switches to a static +slate and streams forever. +""" +import argparse +import contextlib +import io +import os +import re +import subprocess +import sys +import time +from datetime import datetime + +ROOT = os.path.dirname(os.path.abspath(__file__)) +RECORDINGS = os.path.join(ROOT, "recordings") +LOGS = os.path.join(ROOT, "logs") +LIVE_URL = "https://www.bristolrovers.co.uk/video/live" +UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " + "Gecko/20100101 Firefox/131.0") +COOKIE_DOMAINS = ("bristolrovers", "streamamg", "kaltura", "akamaized") + + +def log(msg): + line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}" + print(line, flush=True) + os.makedirs(LOGS, exist_ok=True) + with open(os.path.join(LOGS, "record_live.log"), "a", encoding="utf-8") as fh: + fh.write(line + chr(10)) + + +def firefox_cookies(): + from yt_dlp.cookies import extract_cookies_from_browser + with contextlib.redirect_stderr(io.StringIO()): + jar = extract_cookies_from_browser("firefox") + out = [] + for c in jar: + dom = c.domain or "" + if not any(k in dom.lower() for k in COOKIE_DOMAINS): + continue + out.append({"name": c.name, "value": c.value or "", "domain": dom, + "path": c.path or "/", + "expires": float(c.expires) if c.expires else -1, + "httpOnly": False, "secure": bool(c.secure), + "sameSite": "Lax"}) + return out + + +TOKEN_URL = ("https://www.bristolrovers.co.uk/oauth/browser-token" + "?client_id=streamamg&token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Aid_token") +# Public client key, shipped to every browser -- not a secret. +API_KEY = "50ivbQ3IXpF2mwT4tPAJ2V6cVm4CGRT2YKmDK5v6" + + +def mint_master(entry_id): + """Mint a fresh master playlist URL in plain Python. No browser. + + Handing yt-dlp the *media* playlist (hdntl, path token) fails: Python + clients get 403 on that URL form, and yt-dlp fetches the playlist with + urllib before ffmpeg ever runs. The *master* (hdnts, query token) is fetched + fine by Python, so we mint one here and pass that instead. It lives only + ~25 seconds, which is why it is minted immediately before recording. + """ + import http.cookiejar, json, re, urllib.request + from yt_dlp.cookies import extract_cookies_from_browser + + with contextlib.redirect_stderr(io.StringIO()): + jar = extract_cookies_from_browser("firefox") + cj = http.cookiejar.CookieJar() + for c in jar: + if "bristolrovers" in (c.domain or "").lower(): + cj.set_cookie(c) + op = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) + hdrs = {"User-Agent": UA, "Accept": "application/json, text/plain, */*"} + + body = op.open(urllib.request.Request(TOKEN_URL, headers=hdrs), + timeout=20).read().decode("utf-8", "replace") + m = re.search(r"(eyJ[\w-]+\.eyJ[\w-]+\.[\w-]+)", body) + if not m: + log(" mint: no JWT in browser-token response (session expired?)") + return None + jwt = m.group(1) + + req = urllib.request.Request( + f"https://api.playback.streamamg.com/v1/entry/{entry_id}", + headers={**hdrs, "Authorization": "Bearer " + jwt, "x-api-key": API_KEY, + "Origin": "https://www.bristolrovers.co.uk", + "Referer": "https://www.bristolrovers.co.uk/"}) + data = json.loads(urllib.request.urlopen(req, timeout=20) + .read().decode("utf-8", "replace")) + return (data.get("media") or {}).get("hls") or None + + +def find_manifest(timeout_s=90): + """Load the live page headlessly and return the entry id, or None. + + The browser is needed only for this: the entry id appears at runtime, and + the static HTML carries none when nothing is live. Everything after this + is plain HTTP. + """ + from playwright.sync_api import sync_playwright + + found = {"master": None, "media": None} + + with sync_playwright() as p: + browser = p.firefox.launch(headless=True) + ctx = browser.new_context(user_agent=UA, + viewport={"width": 1440, "height": 900}) + ctx.add_cookies(firefox_cookies()) + page = ctx.new_page() + + def on_response(resp): + u = resp.url + if ".m3u8" not in u.split("?")[0].lower(): + return + if "hdntl=" in u and not found["media"]: + found["media"] = u + elif "hdnts=" in u and not found["master"]: + found["master"] = u + + page.on("response", on_response) + try: + page.goto(LIVE_URL, wait_until="domcontentloaded", timeout=60000) + except Exception as exc: + log(f" page load failed: {exc}") + browser.close() + return None + + # The manifest is fetched on load, before any click. We still click, + # because a stream that has only just gone live may need the nudge. + deadline = time.time() + timeout_s + clicked = False + while time.time() < deadline and not found["media"]: + if not clicked: + for sel in (".bmpui-ui-hugeplaybacktogglebutton", + ".bmpui-ui-playbacktogglebutton", + "button[aria-label*='play' i]"): + try: + el = page.query_selector(sel) + if el and el.is_visible(): + el.click() + clicked = True + break + except Exception: + pass + with contextlib.suppress(Exception): + page.evaluate("() => { const v=document.querySelector('video');" + " if (v) { v.muted = true; v.play(); } }") + page.wait_for_timeout(2000) + + browser.close() + + seen_url = found["media"] or found["master"] + if not seen_url: + return None + m = re.search(r"/(\d+_[a-z0-9]+)/", seen_url, re.I) + return m.group(1) if m else None + + +def record(url, duration_min): + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out = os.path.join(RECORDINGS, f"brfc-{stamp}.ts") + os.makedirs(RECORDINGS, exist_ok=True) + logfile = os.path.join(LOGS, f"yt-dlp-{stamp}.log") + cmd = [ + sys.executable, "-m", "yt_dlp", + # Force ffmpeg. yt-dlp's native HLS downloader fetches variant + # playlists with Python and gets 403/404 on the path-token URLs; + # ffmpeg fetches them fine. This is not a preference, it is required. + "--downloader", "ffmpeg", + "--hls-use-mpegts", # a killed process still leaves a playable file + "--no-part", + "--cookies-from-browser", "firefox", + "--user-agent", UA, + "--referer", "https://www.bristolrovers.co.uk/", + "--downloader-args", + # ffmpeg 4.3.1 has no -reconnect_on_network_error. + "ffmpeg_i:-reconnect 1 -reconnect_streamed 1 " + "-reconnect_delay_max 30", + "--downloader-args", f"ffmpeg_o:-t {duration_min * 60}", + "-o", out, + url, + ] + log(f"recording -> {os.path.basename(out)} (cap {duration_min} min)") + log(f" yt-dlp log -> {os.path.basename(logfile)}") + with open(logfile, "w", encoding="utf-8") as fh: + return subprocess.Popen(cmd, stdout=fh, stderr=subprocess.STDOUT, cwd=ROOT) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--at", help='wall-clock start, e.g. "2026-09-02 03:15"') + ap.add_argument("--max-wait", type=int, default=45, + help="minutes to keep looking for a live stream") + ap.add_argument("--duration", type=int, default=150, + help="recording cap in minutes") + ap.add_argument("--poll", type=int, default=60, help="seconds between attempts") + args = ap.parse_args() + + if args.at: + target = datetime.strptime(args.at, "%Y-%m-%d %H:%M") + wait = (target - datetime.now()).total_seconds() + if wait > 0: + log(f"sleeping {wait/3600:.2f}h until {target:%Y-%m-%d %H:%M}") + time.sleep(wait) + + deadline = time.time() + args.max_wait * 60 + attempt = 0 + while time.time() < deadline: + attempt += 1 + log(f"attempt {attempt}: looking for a live stream") + try: + entry_id = find_manifest() + except Exception as exc: + log(f" error: {type(exc).__name__}: {exc}") + entry_id = None + if entry_id: + log(f" live entry: {entry_id}") + try: + url = mint_master(entry_id) + except Exception as exc: + log(f" mint failed: {type(exc).__name__}: {exc}") + url = None + if not url: + log(" could not mint a manifest; retrying") + time.sleep(args.poll) + continue + log(" minted fresh master playlist") + proc = record(url, args.duration) + log(f" downloader pid {proc.pid}; waiting for it to finish") + t0 = time.time() + proc.wait() + ran = time.time() - t0 + log(f" downloader exited rc={proc.returncode} after {ran:.0f}s") + + # A master playlist outlives its stream: after a match ends the + # master is still served while its variants 404. So "found a + # master" does not mean "stream is live" -- only a download that + # actually ran does. Anything that dies immediately means the + # stream is not up yet, so keep waiting rather than giving up. + if proc.returncode == 0 or ran > 120: + return 0 + log(" died immediately -- stream not live yet, will retry") + time.sleep(args.poll) + continue + log(f" nothing live yet; retrying in {args.poll}s") + time.sleep(args.poll) + + log("gave up: no live stream appeared within the wait window") + return 1 + + +if __name__ == "__main__": + sys.exit(main())