diff --git a/catcher.py b/catcher.py index c7e8cfa..66b6c63 100644 --- a/catcher.py +++ b/catcher.py @@ -68,8 +68,9 @@ def start_recording(key, url, user_agent, page): # yt-dlp's own retry flags do NOT reach the ffmpeg downloader, which is # what actually pulls live HLS. These are ffmpeg's own reconnect knobs. "--downloader-args", + # ffmpeg 4.3.1 has no -reconnect_on_network_error. "ffmpeg_i:-reconnect 1 -reconnect_streamed 1 " - "-reconnect_on_network_error 1 -reconnect_delay_max 30", + "-reconnect_delay_max 30", "--downloader-args", f"ffmpeg_o:-t {DURATION_MIN * 60}", "-o", out, url, diff --git a/record_live.py b/record_live.py new file mode 100644 index 0000000..6c267f4 --- /dev/null +++ b/record_live.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Unattended recorder: find the live stream, then record it. + +Why this needs a headless browser at all: auth and the token mint are plain +HTTP (see discover.py), but the *entry id* for a given match only appears in +the page at runtime -- the static HTML carries no id when nothing is live. So +we render the page headlessly just long enough to learn the manifest URL, then +hand it to ffmpeg and close the browser. + +Nothing here needs your password. It rides the Firefox session cookies, which +outlive kickoff by months. + + # start now, keep looking for up to 45 min, then record 150 min + python record_live.py --max-wait 45 --duration 150 + + # or wait until a wall-clock time first (03:15 before an 03:45 kickoff) + python record_live.py --at "2026-09-02 03:15" --max-wait 60 + +Recording stops on the duration cap. That is deliberate and load-bearing: the +feed does not emit #EXT-X-ENDLIST -- after full time it switches to a static +slate and streams forever. +""" +import argparse +import contextlib +import io +import os +import re +import subprocess +import sys +import time +from datetime import datetime + +ROOT = os.path.dirname(os.path.abspath(__file__)) +RECORDINGS = os.path.join(ROOT, "recordings") +LOGS = os.path.join(ROOT, "logs") +LIVE_URL = "https://www.bristolrovers.co.uk/video/live" +UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " + "Gecko/20100101 Firefox/131.0") +COOKIE_DOMAINS = ("bristolrovers", "streamamg", "kaltura", "akamaized") + + +def log(msg): + line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}" + print(line, flush=True) + os.makedirs(LOGS, exist_ok=True) + with open(os.path.join(LOGS, "record_live.log"), "a", encoding="utf-8") as fh: + fh.write(line + chr(10)) + + +def firefox_cookies(): + from yt_dlp.cookies import extract_cookies_from_browser + with contextlib.redirect_stderr(io.StringIO()): + jar = extract_cookies_from_browser("firefox") + out = [] + for c in jar: + dom = c.domain or "" + if not any(k in dom.lower() for k in COOKIE_DOMAINS): + continue + out.append({"name": c.name, "value": c.value or "", "domain": dom, + "path": c.path or "/", + "expires": float(c.expires) if c.expires else -1, + "httpOnly": False, "secure": bool(c.secure), + "sameSite": "Lax"}) + return out + + +TOKEN_URL = ("https://www.bristolrovers.co.uk/oauth/browser-token" + "?client_id=streamamg&token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Aid_token") +# Public client key, shipped to every browser -- not a secret. +API_KEY = "50ivbQ3IXpF2mwT4tPAJ2V6cVm4CGRT2YKmDK5v6" + + +def mint_master(entry_id): + """Mint a fresh master playlist URL in plain Python. No browser. + + Handing yt-dlp the *media* playlist (hdntl, path token) fails: Python + clients get 403 on that URL form, and yt-dlp fetches the playlist with + urllib before ffmpeg ever runs. The *master* (hdnts, query token) is fetched + fine by Python, so we mint one here and pass that instead. It lives only + ~25 seconds, which is why it is minted immediately before recording. + """ + import http.cookiejar, json, re, urllib.request + from yt_dlp.cookies import extract_cookies_from_browser + + with contextlib.redirect_stderr(io.StringIO()): + jar = extract_cookies_from_browser("firefox") + cj = http.cookiejar.CookieJar() + for c in jar: + if "bristolrovers" in (c.domain or "").lower(): + cj.set_cookie(c) + op = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) + hdrs = {"User-Agent": UA, "Accept": "application/json, text/plain, */*"} + + body = op.open(urllib.request.Request(TOKEN_URL, headers=hdrs), + timeout=20).read().decode("utf-8", "replace") + m = re.search(r"(eyJ[\w-]+\.eyJ[\w-]+\.[\w-]+)", body) + if not m: + log(" mint: no JWT in browser-token response (session expired?)") + return None + jwt = m.group(1) + + req = urllib.request.Request( + f"https://api.playback.streamamg.com/v1/entry/{entry_id}", + headers={**hdrs, "Authorization": "Bearer " + jwt, "x-api-key": API_KEY, + "Origin": "https://www.bristolrovers.co.uk", + "Referer": "https://www.bristolrovers.co.uk/"}) + data = json.loads(urllib.request.urlopen(req, timeout=20) + .read().decode("utf-8", "replace")) + return (data.get("media") or {}).get("hls") or None + + +def find_manifest(timeout_s=90): + """Load the live page headlessly and return the entry id, or None. + + The browser is needed only for this: the entry id appears at runtime, and + the static HTML carries none when nothing is live. Everything after this + is plain HTTP. + """ + from playwright.sync_api import sync_playwright + + found = {"master": None, "media": None} + + with sync_playwright() as p: + browser = p.firefox.launch(headless=True) + ctx = browser.new_context(user_agent=UA, + viewport={"width": 1440, "height": 900}) + ctx.add_cookies(firefox_cookies()) + page = ctx.new_page() + + def on_response(resp): + u = resp.url + if ".m3u8" not in u.split("?")[0].lower(): + return + if "hdntl=" in u and not found["media"]: + found["media"] = u + elif "hdnts=" in u and not found["master"]: + found["master"] = u + + page.on("response", on_response) + try: + page.goto(LIVE_URL, wait_until="domcontentloaded", timeout=60000) + except Exception as exc: + log(f" page load failed: {exc}") + browser.close() + return None + + # The manifest is fetched on load, before any click. We still click, + # because a stream that has only just gone live may need the nudge. + deadline = time.time() + timeout_s + clicked = False + while time.time() < deadline and not found["media"]: + if not clicked: + for sel in (".bmpui-ui-hugeplaybacktogglebutton", + ".bmpui-ui-playbacktogglebutton", + "button[aria-label*='play' i]"): + try: + el = page.query_selector(sel) + if el and el.is_visible(): + el.click() + clicked = True + break + except Exception: + pass + with contextlib.suppress(Exception): + page.evaluate("() => { const v=document.querySelector('video');" + " if (v) { v.muted = true; v.play(); } }") + page.wait_for_timeout(2000) + + browser.close() + + seen_url = found["media"] or found["master"] + if not seen_url: + return None + m = re.search(r"/(\d+_[a-z0-9]+)/", seen_url, re.I) + return m.group(1) if m else None + + +def record(url, duration_min): + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out = os.path.join(RECORDINGS, f"brfc-{stamp}.ts") + os.makedirs(RECORDINGS, exist_ok=True) + logfile = os.path.join(LOGS, f"yt-dlp-{stamp}.log") + cmd = [ + sys.executable, "-m", "yt_dlp", + # Force ffmpeg. yt-dlp's native HLS downloader fetches variant + # playlists with Python and gets 403/404 on the path-token URLs; + # ffmpeg fetches them fine. This is not a preference, it is required. + "--downloader", "ffmpeg", + "--hls-use-mpegts", # a killed process still leaves a playable file + "--no-part", + "--cookies-from-browser", "firefox", + "--user-agent", UA, + "--referer", "https://www.bristolrovers.co.uk/", + "--downloader-args", + # ffmpeg 4.3.1 has no -reconnect_on_network_error. + "ffmpeg_i:-reconnect 1 -reconnect_streamed 1 " + "-reconnect_delay_max 30", + "--downloader-args", f"ffmpeg_o:-t {duration_min * 60}", + "-o", out, + url, + ] + log(f"recording -> {os.path.basename(out)} (cap {duration_min} min)") + log(f" yt-dlp log -> {os.path.basename(logfile)}") + with open(logfile, "w", encoding="utf-8") as fh: + return subprocess.Popen(cmd, stdout=fh, stderr=subprocess.STDOUT, cwd=ROOT) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--at", help='wall-clock start, e.g. "2026-09-02 03:15"') + ap.add_argument("--max-wait", type=int, default=45, + help="minutes to keep looking for a live stream") + ap.add_argument("--duration", type=int, default=150, + help="recording cap in minutes") + ap.add_argument("--poll", type=int, default=60, help="seconds between attempts") + args = ap.parse_args() + + if args.at: + target = datetime.strptime(args.at, "%Y-%m-%d %H:%M") + wait = (target - datetime.now()).total_seconds() + if wait > 0: + log(f"sleeping {wait/3600:.2f}h until {target:%Y-%m-%d %H:%M}") + time.sleep(wait) + + deadline = time.time() + args.max_wait * 60 + attempt = 0 + while time.time() < deadline: + attempt += 1 + log(f"attempt {attempt}: looking for a live stream") + try: + entry_id = find_manifest() + except Exception as exc: + log(f" error: {type(exc).__name__}: {exc}") + entry_id = None + if entry_id: + log(f" live entry: {entry_id}") + try: + url = mint_master(entry_id) + except Exception as exc: + log(f" mint failed: {type(exc).__name__}: {exc}") + url = None + if not url: + log(" could not mint a manifest; retrying") + time.sleep(args.poll) + continue + log(" minted fresh master playlist") + proc = record(url, args.duration) + log(f" downloader pid {proc.pid}; waiting for it to finish") + t0 = time.time() + proc.wait() + ran = time.time() - t0 + log(f" downloader exited rc={proc.returncode} after {ran:.0f}s") + + # A master playlist outlives its stream: after a match ends the + # master is still served while its variants 404. So "found a + # master" does not mean "stream is live" -- only a download that + # actually ran does. Anything that dies immediately means the + # stream is not up yet, so keep waiting rather than giving up. + if proc.returncode == 0 or ran > 120: + return 0 + log(" died immediately -- stream not live yet, will retry") + time.sleep(args.poll) + continue + log(f" nothing live yet; retrying in {args.poll}s") + time.sleep(args.poll) + + log("gave up: no live stream appeared within the wait window") + return 1 + + +if __name__ == "__main__": + sys.exit(main())