Mikkeli Matlock
14ee5ec26c
Two failure modes cost a match silently, and neither was detectable from inside the recorder. preflight.py: the session can be dead while every local signal looks fine -- right cookie, right expiry, right profile. Only the server knows. This spends one request asking it to mint a JWT and exits non-zero if it will not, so a dead login surfaces hours before kickoff instead of at it. Reports cookies and tokens by length; it never prints a value. watchdog.py: a scheduled launch has been seen to hang at interpreter startup, alive but frozen before any of our code runs. So "is a recorder process running" is the wrong question -- the hung process answers yes. This asks whether recording is *progressing*, via a growing .ts or a freshly written log, and only intervenes when both signals are stale. A healthy capture is never interrupted. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Fr1fLMnVGNAKH5L2BmhMxn
129 lines
4.8 KiB
Python
129 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Backup starter: confirm something is actually recording, else start it.
|
|
|
|
Schedule this a few minutes before kickoff, behind the primary recording task.
|
|
It exists because a scheduled launch of record_live.py has been seen to hang
|
|
dead at interpreter startup -- process alive, zero CPU, nothing logged, forever.
|
|
Unattended, that loses the match silently.
|
|
|
|
So "is a recorder process running" is the wrong question: the hung process
|
|
answers yes. This asks whether recording is *progressing*:
|
|
|
|
1. a .ts in recordings/ grew in the last --quiet-secs -> healthy, leave it
|
|
2. record_live.log was written in the last --quiet-secs -> mid search loop,
|
|
leave it
|
|
3. neither -> stale. Kill any
|
|
recorder process
|
|
and start a fresh
|
|
one.
|
|
|
|
Killing only ever targets a recorder that has gone quiet on both signals, so a
|
|
healthy capture is never interrupted.
|
|
|
|
python watchdog.py --duration 170 --max-wait 55
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime
|
|
|
|
ROOT = os.path.dirname(os.path.abspath(__file__))
|
|
RECORDINGS = os.path.join(ROOT, "recordings")
|
|
LOGS = os.path.join(ROOT, "logs")
|
|
RECORDER = os.path.join(ROOT, "record_live.py")
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}"
|
|
print(line, flush=True)
|
|
os.makedirs(LOGS, exist_ok=True)
|
|
with open(os.path.join(LOGS, "watchdog.log"), "a", encoding="utf-8") as fh:
|
|
fh.write(line + chr(10))
|
|
|
|
|
|
def powershell(cmd):
|
|
try:
|
|
out = subprocess.run(["powershell", "-NoProfile", "-Command", cmd],
|
|
capture_output=True, text=True, timeout=60)
|
|
return out.stdout.strip()
|
|
except Exception as exc:
|
|
log(f" powershell failed: {type(exc).__name__}: {exc}")
|
|
return ""
|
|
|
|
|
|
def recorder_pids():
|
|
"""PIDs of any running record_live.py, hung or healthy."""
|
|
out = powershell(
|
|
"Get-CimInstance Win32_Process -Filter \"Name='python.exe'\" | "
|
|
"Where-Object { $_.CommandLine -match 'record_live' } | "
|
|
"ForEach-Object { $_.ProcessId }")
|
|
return [int(x) for x in out.split() if x.strip().isdigit()]
|
|
|
|
|
|
def newest_mtime(pattern):
|
|
files = glob.glob(pattern)
|
|
return max((os.path.getmtime(f) for f in files), default=0)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--duration", type=int, default=170,
|
|
help="recording cap in minutes for a run we start")
|
|
ap.add_argument("--max-wait", type=int, default=55,
|
|
help="minutes to keep looking for a live stream")
|
|
ap.add_argument("--prefer", choices=("best", "audio", "video"), default="best")
|
|
ap.add_argument("--quiet-secs", type=int, default=180,
|
|
help="how stale both signals must be before we intervene")
|
|
args = ap.parse_args()
|
|
|
|
log("watchdog: checking whether anything is recording")
|
|
now = time.time()
|
|
|
|
ts_age = now - newest_mtime(os.path.join(RECORDINGS, "*.ts"))
|
|
log_age = now - newest_mtime(os.path.join(LOGS, "record_live.log"))
|
|
pids = recorder_pids()
|
|
log(f" recorder pids {pids or 'none'}; newest .ts {ts_age:.0f}s old; "
|
|
f"record_live.log {log_age:.0f}s old")
|
|
|
|
if ts_age < args.quiet_secs:
|
|
log(" a recording is growing -- healthy, nothing to do")
|
|
return 0
|
|
if pids and log_age < args.quiet_secs:
|
|
log(" recorder is logging -- mid search loop, nothing to do")
|
|
return 0
|
|
|
|
if pids:
|
|
log(f" recorder {pids} is alive but both signals are stale -- killing it")
|
|
for pid in pids:
|
|
powershell(f"Stop-Process -Id {pid} -Force -ErrorAction SilentlyContinue")
|
|
time.sleep(3)
|
|
still = recorder_pids()
|
|
if still:
|
|
log(f" WARNING: {still} survived the kill; starting anyway")
|
|
else:
|
|
log(" nothing is recording")
|
|
|
|
cmd = [sys.executable, RECORDER,
|
|
"--max-wait", str(args.max_wait),
|
|
"--duration", str(args.duration),
|
|
"--prefer", args.prefer]
|
|
log(f" starting: {' '.join(cmd[1:])}")
|
|
# Detached: the watchdog task exits immediately, the recorder outlives it.
|
|
flags = 0
|
|
if os.name == "nt":
|
|
flags = subprocess.CREATE_NEW_PROCESS_GROUP | getattr(
|
|
subprocess, "DETACHED_PROCESS", 0)
|
|
proc = subprocess.Popen(cmd, cwd=ROOT, creationflags=flags,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL)
|
|
log(f" started recorder pid {proc.pid}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|