Add a pre-match session check and a recovery watchdog

Two failure modes cost a match silently, and neither was detectable from
inside the recorder.

preflight.py: the session can be dead while every local signal looks fine --
right cookie, right expiry, right profile. Only the server knows. This spends
one request asking it to mint a JWT and exits non-zero if it will not, so a
dead login surfaces hours before kickoff instead of at it. Reports cookies and
tokens by length; it never prints a value.

watchdog.py: a scheduled launch has been seen to hang at interpreter startup,
alive but frozen before any of our code runs. So "is a recorder process
running" is the wrong question -- the hung process answers yes. This asks
whether recording is *progressing*, via a growing .ts or a freshly written
log, and only intervenes when both signals are stale. A healthy capture is
never interrupted.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Fr1fLMnVGNAKH5L2BmhMxn
This commit is contained in:
Mikkeli Matlock
2026-09-19 21:47:19 +09:00
parent fcf509fda5
commit 14ee5ec26c
2 changed files with 352 additions and 0 deletions
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""Backup starter: confirm something is actually recording, else start it.
Schedule this a few minutes before kickoff, behind the primary recording task.
It exists because a scheduled launch of record_live.py has been seen to hang
dead at interpreter startup -- process alive, zero CPU, nothing logged, forever.
Unattended, that loses the match silently.
So "is a recorder process running" is the wrong question: the hung process
answers yes. This asks whether recording is *progressing*:
1. a .ts in recordings/ grew in the last --quiet-secs -> healthy, leave it
2. record_live.log was written in the last --quiet-secs -> mid search loop,
leave it
3. neither -> stale. Kill any
recorder process
and start a fresh
one.
Killing only ever targets a recorder that has gone quiet on both signals, so a
healthy capture is never interrupted.
python watchdog.py --duration 170 --max-wait 55
"""
import argparse
import glob
import os
import subprocess
import sys
import time
from datetime import datetime
ROOT = os.path.dirname(os.path.abspath(__file__))
RECORDINGS = os.path.join(ROOT, "recordings")
LOGS = os.path.join(ROOT, "logs")
RECORDER = os.path.join(ROOT, "record_live.py")
def log(msg):
line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}"
print(line, flush=True)
os.makedirs(LOGS, exist_ok=True)
with open(os.path.join(LOGS, "watchdog.log"), "a", encoding="utf-8") as fh:
fh.write(line + chr(10))
def powershell(cmd):
try:
out = subprocess.run(["powershell", "-NoProfile", "-Command", cmd],
capture_output=True, text=True, timeout=60)
return out.stdout.strip()
except Exception as exc:
log(f" powershell failed: {type(exc).__name__}: {exc}")
return ""
def recorder_pids():
"""PIDs of any running record_live.py, hung or healthy."""
out = powershell(
"Get-CimInstance Win32_Process -Filter \"Name='python.exe'\" | "
"Where-Object { $_.CommandLine -match 'record_live' } | "
"ForEach-Object { $_.ProcessId }")
return [int(x) for x in out.split() if x.strip().isdigit()]
def newest_mtime(pattern):
files = glob.glob(pattern)
return max((os.path.getmtime(f) for f in files), default=0)
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--duration", type=int, default=170,
help="recording cap in minutes for a run we start")
ap.add_argument("--max-wait", type=int, default=55,
help="minutes to keep looking for a live stream")
ap.add_argument("--prefer", choices=("best", "audio", "video"), default="best")
ap.add_argument("--quiet-secs", type=int, default=180,
help="how stale both signals must be before we intervene")
args = ap.parse_args()
log("watchdog: checking whether anything is recording")
now = time.time()
ts_age = now - newest_mtime(os.path.join(RECORDINGS, "*.ts"))
log_age = now - newest_mtime(os.path.join(LOGS, "record_live.log"))
pids = recorder_pids()
log(f" recorder pids {pids or 'none'}; newest .ts {ts_age:.0f}s old; "
f"record_live.log {log_age:.0f}s old")
if ts_age < args.quiet_secs:
log(" a recording is growing -- healthy, nothing to do")
return 0
if pids and log_age < args.quiet_secs:
log(" recorder is logging -- mid search loop, nothing to do")
return 0
if pids:
log(f" recorder {pids} is alive but both signals are stale -- killing it")
for pid in pids:
powershell(f"Stop-Process -Id {pid} -Force -ErrorAction SilentlyContinue")
time.sleep(3)
still = recorder_pids()
if still:
log(f" WARNING: {still} survived the kill; starting anyway")
else:
log(" nothing is recording")
cmd = [sys.executable, RECORDER,
"--max-wait", str(args.max_wait),
"--duration", str(args.duration),
"--prefer", args.prefer]
log(f" starting: {' '.join(cmd[1:])}")
# Detached: the watchdog task exits immediately, the recorder outlives it.
flags = 0
if os.name == "nt":
flags = subprocess.CREATE_NEW_PROCESS_GROUP | getattr(
subprocess, "DETACHED_PROCESS", 0)
proc = subprocess.Popen(cmd, cwd=ROOT, creationflags=flags,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
log(f" started recorder pid {proc.pid}")
return 0
if __name__ == "__main__":
sys.exit(main())