#!/usr/bin/env python3 """Backup starter: confirm something is actually recording, else start it. Schedule this a few minutes before kickoff, behind the primary recording task. It exists because a scheduled launch of record_live.py has been seen to hang dead at interpreter startup -- process alive, zero CPU, nothing logged, forever. Unattended, that loses the match silently. So "is a recorder process running" is the wrong question: the hung process answers yes. This asks whether recording is *progressing*: 1. a .ts in recordings/ grew in the last --quiet-secs -> healthy, leave it 2. record_live.log was written in the last --quiet-secs -> mid search loop, leave it 3. neither -> stale. Kill any recorder process and start a fresh one. Killing only ever targets a recorder that has gone quiet on both signals, so a healthy capture is never interrupted. python watchdog.py --duration 170 --max-wait 55 """ import argparse import glob import os import subprocess import sys import time from datetime import datetime ROOT = os.path.dirname(os.path.abspath(__file__)) RECORDINGS = os.path.join(ROOT, "recordings") LOGS = os.path.join(ROOT, "logs") RECORDER = os.path.join(ROOT, "record_live.py") def log(msg): line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}" print(line, flush=True) os.makedirs(LOGS, exist_ok=True) with open(os.path.join(LOGS, "watchdog.log"), "a", encoding="utf-8") as fh: fh.write(line + chr(10)) def powershell(cmd): try: out = subprocess.run(["powershell", "-NoProfile", "-Command", cmd], capture_output=True, text=True, timeout=60) return out.stdout.strip() except Exception as exc: log(f" powershell failed: {type(exc).__name__}: {exc}") return "" def recorder_pids(): """PIDs of any running record_live.py, hung or healthy.""" out = powershell( "Get-CimInstance Win32_Process -Filter \"Name='python.exe'\" | " "Where-Object { $_.CommandLine -match 'record_live' } | " "ForEach-Object { $_.ProcessId }") return [int(x) for x in out.split() if x.strip().isdigit()] def newest_mtime(pattern): files = glob.glob(pattern) return max((os.path.getmtime(f) for f in files), default=0) def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--duration", type=int, default=170, help="recording cap in minutes for a run we start") ap.add_argument("--max-wait", type=int, default=55, help="minutes to keep looking for a live stream") ap.add_argument("--prefer", choices=("best", "audio", "video"), default="best") ap.add_argument("--quiet-secs", type=int, default=180, help="how stale both signals must be before we intervene") args = ap.parse_args() log("watchdog: checking whether anything is recording") now = time.time() ts_age = now - newest_mtime(os.path.join(RECORDINGS, "*.ts")) log_age = now - newest_mtime(os.path.join(LOGS, "record_live.log")) pids = recorder_pids() log(f" recorder pids {pids or 'none'}; newest .ts {ts_age:.0f}s old; " f"record_live.log {log_age:.0f}s old") if ts_age < args.quiet_secs: log(" a recording is growing -- healthy, nothing to do") return 0 if pids and log_age < args.quiet_secs: log(" recorder is logging -- mid search loop, nothing to do") return 0 if pids: log(f" recorder {pids} is alive but both signals are stale -- killing it") for pid in pids: powershell(f"Stop-Process -Id {pid} -Force -ErrorAction SilentlyContinue") time.sleep(3) still = recorder_pids() if still: log(f" WARNING: {still} survived the kill; starting anyway") else: log(" nothing is recording") cmd = [sys.executable, RECORDER, "--max-wait", str(args.max_wait), "--duration", str(args.duration), "--prefer", args.prefer] log(f" starting: {' '.join(cmd[1:])}") # Detached: the watchdog task exits immediately, the recorder outlives it. flags = 0 if os.name == "nt": flags = subprocess.CREATE_NEW_PROCESS_GROUP | getattr( subprocess, "DETACHED_PROCESS", 0) proc = subprocess.Popen(cmd, cwd=ROOT, creationflags=flags, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) log(f" started recorder pid {proc.pid}") return 0 if __name__ == "__main__": sys.exit(main())