Add a pre-match session check and a recovery watchdog
Two failure modes cost a match silently, and neither was detectable from inside the recorder. preflight.py: the session can be dead while every local signal looks fine -- right cookie, right expiry, right profile. Only the server knows. This spends one request asking it to mint a JWT and exits non-zero if it will not, so a dead login surfaces hours before kickoff instead of at it. Reports cookies and tokens by length; it never prints a value. watchdog.py: a scheduled launch has been seen to hang at interpreter startup, alive but frozen before any of our code runs. So "is a recorder process running" is the wrong question -- the hung process answers yes. This asks whether recording is *progressing*, via a growing .ts or a freshly written log, and only intervenes when both signals are stale. A healthy capture is never interrupted. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Fr1fLMnVGNAKH5L2BmhMxn
This commit is contained in:
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backup starter: confirm something is actually recording, else start it.
|
||||
|
||||
Schedule this a few minutes before kickoff, behind the primary recording task.
|
||||
It exists because a scheduled launch of record_live.py has been seen to hang
|
||||
dead at interpreter startup -- process alive, zero CPU, nothing logged, forever.
|
||||
Unattended, that loses the match silently.
|
||||
|
||||
So "is a recorder process running" is the wrong question: the hung process
|
||||
answers yes. This asks whether recording is *progressing*:
|
||||
|
||||
1. a .ts in recordings/ grew in the last --quiet-secs -> healthy, leave it
|
||||
2. record_live.log was written in the last --quiet-secs -> mid search loop,
|
||||
leave it
|
||||
3. neither -> stale. Kill any
|
||||
recorder process
|
||||
and start a fresh
|
||||
one.
|
||||
|
||||
Killing only ever targets a recorder that has gone quiet on both signals, so a
|
||||
healthy capture is never interrupted.
|
||||
|
||||
python watchdog.py --duration 170 --max-wait 55
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
RECORDINGS = os.path.join(ROOT, "recordings")
|
||||
LOGS = os.path.join(ROOT, "logs")
|
||||
RECORDER = os.path.join(ROOT, "record_live.py")
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}"
|
||||
print(line, flush=True)
|
||||
os.makedirs(LOGS, exist_ok=True)
|
||||
with open(os.path.join(LOGS, "watchdog.log"), "a", encoding="utf-8") as fh:
|
||||
fh.write(line + chr(10))
|
||||
|
||||
|
||||
def powershell(cmd):
|
||||
try:
|
||||
out = subprocess.run(["powershell", "-NoProfile", "-Command", cmd],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
return out.stdout.strip()
|
||||
except Exception as exc:
|
||||
log(f" powershell failed: {type(exc).__name__}: {exc}")
|
||||
return ""
|
||||
|
||||
|
||||
def recorder_pids():
|
||||
"""PIDs of any running record_live.py, hung or healthy."""
|
||||
out = powershell(
|
||||
"Get-CimInstance Win32_Process -Filter \"Name='python.exe'\" | "
|
||||
"Where-Object { $_.CommandLine -match 'record_live' } | "
|
||||
"ForEach-Object { $_.ProcessId }")
|
||||
return [int(x) for x in out.split() if x.strip().isdigit()]
|
||||
|
||||
|
||||
def newest_mtime(pattern):
|
||||
files = glob.glob(pattern)
|
||||
return max((os.path.getmtime(f) for f in files), default=0)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--duration", type=int, default=170,
|
||||
help="recording cap in minutes for a run we start")
|
||||
ap.add_argument("--max-wait", type=int, default=55,
|
||||
help="minutes to keep looking for a live stream")
|
||||
ap.add_argument("--prefer", choices=("best", "audio", "video"), default="best")
|
||||
ap.add_argument("--quiet-secs", type=int, default=180,
|
||||
help="how stale both signals must be before we intervene")
|
||||
args = ap.parse_args()
|
||||
|
||||
log("watchdog: checking whether anything is recording")
|
||||
now = time.time()
|
||||
|
||||
ts_age = now - newest_mtime(os.path.join(RECORDINGS, "*.ts"))
|
||||
log_age = now - newest_mtime(os.path.join(LOGS, "record_live.log"))
|
||||
pids = recorder_pids()
|
||||
log(f" recorder pids {pids or 'none'}; newest .ts {ts_age:.0f}s old; "
|
||||
f"record_live.log {log_age:.0f}s old")
|
||||
|
||||
if ts_age < args.quiet_secs:
|
||||
log(" a recording is growing -- healthy, nothing to do")
|
||||
return 0
|
||||
if pids and log_age < args.quiet_secs:
|
||||
log(" recorder is logging -- mid search loop, nothing to do")
|
||||
return 0
|
||||
|
||||
if pids:
|
||||
log(f" recorder {pids} is alive but both signals are stale -- killing it")
|
||||
for pid in pids:
|
||||
powershell(f"Stop-Process -Id {pid} -Force -ErrorAction SilentlyContinue")
|
||||
time.sleep(3)
|
||||
still = recorder_pids()
|
||||
if still:
|
||||
log(f" WARNING: {still} survived the kill; starting anyway")
|
||||
else:
|
||||
log(" nothing is recording")
|
||||
|
||||
cmd = [sys.executable, RECORDER,
|
||||
"--max-wait", str(args.max_wait),
|
||||
"--duration", str(args.duration),
|
||||
"--prefer", args.prefer]
|
||||
log(f" starting: {' '.join(cmd[1:])}")
|
||||
# Detached: the watchdog task exits immediately, the recorder outlives it.
|
||||
flags = 0
|
||||
if os.name == "nt":
|
||||
flags = subprocess.CREATE_NEW_PROCESS_GROUP | getattr(
|
||||
subprocess, "DETACHED_PROCESS", 0)
|
||||
proc = subprocess.Popen(cmd, cwd=ROOT, creationflags=flags,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
log(f" started recorder pid {proc.pid}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user