fix: stabilize overnight startup and monitor live fallback (#396 #397)

scripts/runtime_verify_monitor.sh

@@ -7,12 +7,14 @@ ROOT_DIR="${ROOT_DIR:-/home/agentson/repos/The-Ouroboros}"
 LOG_DIR="${LOG_DIR:-$ROOT_DIR/data/overnight}"
 INTERVAL_SEC="${INTERVAL_SEC:-60}"
 MAX_HOURS="${MAX_HOURS:-24}"
+MAX_LOOPS="${MAX_LOOPS:-0}"
 POLICY_TZ="${POLICY_TZ:-Asia/Seoul}"
 
 cd "$ROOT_DIR"
 
 OUT_LOG="$LOG_DIR/runtime_verify_$(date +%Y%m%d_%H%M%S).log"
 END_TS=$(( $(date +%s) + MAX_HOURS*3600 ))
+loops=0
 
 log() {
   printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$1" | tee -a "$OUT_LOG" >/dev/null
@@ -31,6 +33,11 @@ check_signal() {
   return 1
 }
 
+find_live_pids() {
+  # Detect live-mode process even when run_overnight pid files are absent.
+  pgrep -af "[s]rc.main --mode=live" 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//'
+}
+
 check_forbidden() {
   local name="$1"
   local pattern="$2"
@@ -47,39 +54,63 @@ check_forbidden() {
 log "[INFO] runtime verify monitor started interval=${INTERVAL_SEC}s max_hours=${MAX_HOURS} policy_tz=${POLICY_TZ}"
 
 while true; do
+  loops=$((loops + 1))
   now=$(date +%s)
   if [ "$now" -ge "$END_TS" ]; then
     log "[INFO] monitor completed (time window reached)"
     exit 0
   fi
+  if [ "$MAX_LOOPS" -gt 0 ] && [ "$loops" -gt "$MAX_LOOPS" ]; then
+    log "[INFO] monitor completed (max loops reached)"
+    exit 0
+  fi
 
   latest_run="$(ls -t "$LOG_DIR"/run_*.log 2>/dev/null | head -n1 || true)"
-  if [ -z "$latest_run" ]; then
-    log "[ANOMALY] no run log found"
-    sleep "$INTERVAL_SEC"
-    continue
-  fi
 
   # Basic liveness hints.
   app_pid="$(cat "$LOG_DIR/app.pid" 2>/dev/null || true)"
   wd_pid="$(cat "$LOG_DIR/watchdog.pid" 2>/dev/null || true)"
+  live_pids="$(find_live_pids)"
   app_alive=0
   wd_alive=0
   port_alive=0
   [ -n "$app_pid" ] && kill -0 "$app_pid" 2>/dev/null && app_alive=1
   [ -n "$wd_pid" ] && kill -0 "$wd_pid" 2>/dev/null && wd_alive=1
+  if [ "$app_alive" -eq 0 ] && [ -n "$live_pids" ]; then
+    app_alive=1
+  fi
   ss -ltnp 2>/dev/null | rg -q ':8080' && port_alive=1
-  log "[HEARTBEAT] run_log=$latest_run app_alive=$app_alive watchdog_alive=$wd_alive port8080=$port_alive"
+  log "[HEARTBEAT] run_log=${latest_run:-none} app_alive=$app_alive watchdog_alive=$wd_alive port8080=$port_alive live_pids=${live_pids:-none}"
+
+  if [ -z "$latest_run" ]; then
+    log "[ANOMALY] no run log found"
+  fi
 
   # Coverage matrix rows (session paths and policy gate evidence).
   not_observed=0
-  check_signal "LIVE_MODE" "Mode: live" "$latest_run" || not_observed=$((not_observed+1))
-  check_signal "KR_LOOP" "Processing market: Korea Exchange" "$latest_run" || not_observed=$((not_observed+1))
-  check_signal "NXT_PATH" "NXT_PRE|NXT_AFTER|session=NXT_" "$latest_run" || not_observed=$((not_observed+1))
-  check_signal "US_PRE_PATH" "US_PRE|session=US_PRE" "$latest_run" || not_observed=$((not_observed+1))
-  check_signal "US_DAY_PATH" "US_DAY|session=US_DAY|Processing market: .*NASDAQ|Processing market: .*NYSE|Processing market: .*AMEX" "$latest_run" || not_observed=$((not_observed+1))
-  check_signal "US_AFTER_PATH" "US_AFTER|session=US_AFTER" "$latest_run" || not_observed=$((not_observed+1))
-  check_signal "ORDER_POLICY_SESSION" "Order policy rejected .*\\[session=" "$latest_run" || not_observed=$((not_observed+1))
+  if [ "$app_alive" -eq 1 ]; then
+    log "[COVERAGE] LIVE_MODE=PASS source=process_liveness"
+  else
+    if [ -n "$latest_run" ]; then
+      check_signal "LIVE_MODE" "Mode: live" "$latest_run" || not_observed=$((not_observed+1))
+    else
+      log "[COVERAGE] LIVE_MODE=NOT_OBSERVED reason=no_run_log_no_live_pid"
+      not_observed=$((not_observed+1))
+    fi
+  fi
+  if [ -n "$latest_run" ]; then
+    check_signal "KR_LOOP" "Processing market: Korea Exchange" "$latest_run" || not_observed=$((not_observed+1))
+    check_signal "NXT_PATH" "NXT_PRE|NXT_AFTER|session=NXT_" "$latest_run" || not_observed=$((not_observed+1))
+    check_signal "US_PRE_PATH" "US_PRE|session=US_PRE" "$latest_run" || not_observed=$((not_observed+1))
+    check_signal "US_DAY_PATH" "US_DAY|session=US_DAY|Processing market: .*NASDAQ|Processing market: .*NYSE|Processing market: .*AMEX" "$latest_run" || not_observed=$((not_observed+1))
+    check_signal "US_AFTER_PATH" "US_AFTER|session=US_AFTER" "$latest_run" || not_observed=$((not_observed+1))
+    check_signal "ORDER_POLICY_SESSION" "Order policy rejected .*\\[session=" "$latest_run" || not_observed=$((not_observed+1))
+  else
+    for missing in KR_LOOP NXT_PATH US_PRE_PATH US_DAY_PATH US_AFTER_PATH ORDER_POLICY_SESSION; do
+      log "[COVERAGE] ${missing}=NOT_OBSERVED reason=no_run_log"
+      not_observed=$((not_observed+1))
+    done
+  fi
 
   if [ "$not_observed" -gt 0 ]; then
     log "[ANOMALY] coverage_not_observed=$not_observed (treat as FAIL)"
@@ -97,9 +128,13 @@ while true; do
 
   if [ "$is_weekend" -eq 1 ]; then
     # Weekend policy: KR regular session loop must never appear.
-    check_forbidden "WEEKEND_KR_SESSION_ACTIVE" \
-      "Market session active: KR|session=KRX_REG|Processing market: Korea Exchange" \
-      "$latest_run" || forbidden_hits=$((forbidden_hits+1))
+    if [ -n "$latest_run" ]; then
+      check_forbidden "WEEKEND_KR_SESSION_ACTIVE" \
+        "Market session active: KR|session=KRX_REG|Processing market: Korea Exchange" \
+        "$latest_run" || forbidden_hits=$((forbidden_hits+1))
+    else
+      log "[FORBIDDEN] WEEKEND_KR_SESSION_ACTIVE=SKIP reason=no_run_log"
+    fi
   else
     log "[FORBIDDEN] WEEKEND_KR_SESSION_ACTIVE=SKIP reason=weekday"
   fi